@@ -0,0 +1,6 @@
|
||||
# FIX CRLF always when developer has not set
|
||||
# Linux/Mac: git config --global core.autocrlf input
|
||||
# Windows: git config --global core.autocrlf true
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
* eol=lf
|
||||
@@ -0,0 +1,59 @@
|
||||
name: Publish image
|
||||
|
||||
# Build and push the feeder image to the GitHub Container Registry (ghcr.io):
|
||||
# * every push to main -> the moving `main` tag
|
||||
# * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1)
|
||||
# The build context is the repo root (the workspace) so the vendored
|
||||
# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under
|
||||
# feeder-plugin/paper-feeder/.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ['v*']
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract image metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=semver,pattern={{major}}
|
||||
type=sha,format=short
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: ./feeder-plugin/musicbrainz-feeder/Dockerfile
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
Generated
+1948
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,9 @@
|
||||
[workspace]
|
||||
resolver = "2"
|
||||
members = ["feeder-plugin/*"]
|
||||
|
||||
[workspace.package]
|
||||
edition = "2021"
|
||||
rust-version = "1.85"
|
||||
license = "MIT"
|
||||
repository = "https://github.com/worph/meta-feeder-musicbrainz"
|
||||
@@ -1 +1,17 @@
|
||||
# meta-feeder-musicbrainz
|
||||
# meta-feeder-musicbrainz
|
||||
|
||||
MusicBrainz identity cards (album / artist), plus the shared MusicBrainz client used by sibling music feeders.
|
||||
|
||||
**Upstream:** MusicBrainz /ws/2 + Cover Art Archive + ListenBrainz
|
||||
**Role:** identity
|
||||
|
||||
A MetaMesh gateway feeder — one upstream, one job. Implements `FeederPlugin`
|
||||
from [`meta-feeder-sdk`](https://github.com/worph/meta-feeder-sdk).
|
||||
|
||||
```bash
|
||||
cargo build --release --bin musicbrainz-feeder
|
||||
cargo test
|
||||
```
|
||||
|
||||
Scope and naming conventions: `docs/project-architecture/feeder-architecture.md`
|
||||
in the meta-root.
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
[package]
|
||||
name = "musicbrainz-feeder"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
publish = false
|
||||
|
||||
# MusicBrainz identity cards (album / artist), plus the shared MusicBrainz client used by sibling music feeders.
|
||||
# Upstream: MusicBrainz /ws/2 + Cover Art Archive + ListenBrainz
|
||||
|
||||
[lib]
|
||||
name = "musicbrainz_feeder"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "musicbrainz-feeder"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
# TEMPORARY: path dep while the SDK's new rev is unpushed. Flip to:
|
||||
# meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", rev = "<sha>" }
|
||||
meta-feeder-sdk = { path = "../../../meta-feeder-sdk" }
|
||||
|
||||
tokio = { version = "1.40", features = ["full"] }
|
||||
axum = { version = "0.7", features = ["json"] }
|
||||
anyhow = "1"
|
||||
async-trait = "0.1"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
reqwest = { version = "0.12", features = ["json", "rustls-tls", "stream"], default-features = false }
|
||||
bytes = "1"
|
||||
futures = "0.3"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
redb = "2"
|
||||
base64 = "0.22"
|
||||
|
||||
[dev-dependencies]
|
||||
wiremock = "0.6"
|
||||
tempfile = "3"
|
||||
tokio = { version = "1.40", features = ["full", "test-util"] }
|
||||
@@ -0,0 +1,44 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# musicbrainz-feeder sidecar. Multi-stage build; build context is this repo's root.
|
||||
# meta-feeder-sdk arrives as a git dependency, so no vendored crate is copied
|
||||
# in — `git` must be present in the builder for cargo to fetch it.
|
||||
|
||||
FROM rust:1.89-slim-bookworm AS builder
|
||||
WORKDIR /build
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
pkg-config libssl-dev ca-certificates git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY Cargo.toml rust-toolchain.toml ./
|
||||
COPY feeder-plugin/ feeder-plugin/
|
||||
|
||||
RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,id=cargo-git,target=/usr/local/cargo/git \
|
||||
--mount=type=cache,id=musicbrainz-feeder-target,target=/build/target \
|
||||
cargo build --release --bin musicbrainz-feeder \
|
||||
&& cp target/release/musicbrainz-feeder /musicbrainz-feeder
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
FROM debian:bookworm-slim AS runtime
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /musicbrainz-feeder /usr/local/bin/musicbrainz-feeder
|
||||
|
||||
RUN useradd --system --uid 10001 --no-create-home metafeeder \
|
||||
&& mkdir -p /data/meta-feeder \
|
||||
&& chown metafeeder:metafeeder /data/meta-feeder
|
||||
|
||||
USER metafeeder
|
||||
WORKDIR /data/meta-feeder
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
ENV META_FEEDER_HTTP_LISTEN=0.0.0.0:8080 \
|
||||
META_FEEDER_STATE_DIR=/data/meta-feeder \
|
||||
RUST_LOG=info
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/musicbrainz-feeder"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,840 @@
|
||||
//! MusicBrainz web-service v2 client.
|
||||
//!
|
||||
//! The music domain's TMDB. Every call is gated by a shared 1 req/s
|
||||
//! [`RateBudget`] and every MBID-keyed response is cached permanently — an
|
||||
//! MBID names an immutable entity, so there is nothing to revalidate.
|
||||
//!
|
||||
//! ## One call, complete cards
|
||||
//!
|
||||
//! The same principle the card feeder documents: a MusicBrainz **search or
|
||||
//! browse response already carries everything a renderable card needs** —
|
||||
//! `id`, `title`, `first-release-date`, and the `artist-credit` (with the
|
||||
//! artist's own MBID). So a browse row costs *one* request, not one plus
|
||||
//! twenty by-id lookups.
|
||||
//!
|
||||
//! That matters far more here than it does for TMDB. At 1 req/s a
|
||||
//! per-item detail fetch would make a 20-card row take twenty seconds, which
|
||||
//! is past every consumer-side idle cutoff — the row would come back empty,
|
||||
//! not slow. The price is a thinner card (no track count, no label), and those
|
||||
//! are exactly the fields nobody reads until the card is clicked, at which
|
||||
//! point [`MusicBrainzClient::tracklist`] fetches them.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::consts::{
|
||||
MB_API_BASE, MB_PAGE_SIZE, MB_RATE_LIMIT_DEFAULT_RETRY_SECS, MB_TIMEOUT_SECS, USER_AGENT,
|
||||
};
|
||||
use meta_feeder_sdk::budget::{Lease, RateBudget};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Wire shapes
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// An artist credit entry. MusicBrainz models a credit as an ordered list of
|
||||
/// `{artist, joinphrase}` so `"Miles Davis feat. John Coltrane"` round-trips;
|
||||
/// the flattened display string is rebuilt by [`ArtistCredit::render`].
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct ArtistCredit {
|
||||
#[serde(default)]
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub joinphrase: String,
|
||||
#[serde(default)]
|
||||
pub artist: Option<CreditedArtist>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct CreditedArtist {
|
||||
#[serde(default)]
|
||||
pub id: String,
|
||||
#[serde(default)]
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// Flatten a credit list into its display string, honouring join phrases.
|
||||
pub fn render_credit(credits: &[ArtistCredit]) -> String {
|
||||
let mut out = String::new();
|
||||
for c in credits {
|
||||
out.push_str(&c.name);
|
||||
out.push_str(&c.joinphrase);
|
||||
}
|
||||
out.trim().to_string()
|
||||
}
|
||||
|
||||
/// The **primary** artist of a credit list — the first entry with a resolved
|
||||
/// artist MBID. This is what an album card is filed under; the full rendered
|
||||
/// credit is what gets displayed.
|
||||
pub fn primary_artist(credits: &[ArtistCredit]) -> Option<&CreditedArtist> {
|
||||
credits
|
||||
.iter()
|
||||
.filter_map(|c| c.artist.as_ref())
|
||||
.find(|a| !a.id.is_empty())
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct Tag {
|
||||
#[serde(default)]
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub count: i64,
|
||||
}
|
||||
|
||||
/// A release-group: the **work**, an album across all its pressings.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct ReleaseGroup {
|
||||
#[serde(default)]
|
||||
pub id: String,
|
||||
#[serde(default)]
|
||||
pub title: String,
|
||||
/// `Album`, `Single`, `EP`, `Broadcast`, `Other`. Absent on sparse entries.
|
||||
#[serde(rename = "primary-type", default)]
|
||||
pub primary_type: Option<String>,
|
||||
/// `Compilation`, `Live`, `Soundtrack`, `Remix`, … Multiple allowed.
|
||||
#[serde(rename = "secondary-types", default)]
|
||||
pub secondary_types: Vec<String>,
|
||||
/// `YYYY`, `YYYY-MM` or `YYYY-MM-DD` — MusicBrainz dates are
|
||||
/// variable-precision and a consumer must not assume full ISO.
|
||||
#[serde(rename = "first-release-date", default)]
|
||||
pub first_release_date: Option<String>,
|
||||
#[serde(rename = "artist-credit", default)]
|
||||
pub artist_credit: Vec<ArtistCredit>,
|
||||
#[serde(default)]
|
||||
pub tags: Vec<Tag>,
|
||||
#[serde(default)]
|
||||
pub genres: Vec<Tag>,
|
||||
#[serde(default)]
|
||||
pub disambiguation: Option<String>,
|
||||
/// Lucene relevance, 0–100, present only on *search* responses.
|
||||
///
|
||||
/// ⚠ **Saturated, and therefore nearly useless alone.** Measured on
|
||||
/// `query=kind of blue`: six release-groups scored exactly 100 — TQ, Swiss
|
||||
/// Blues Authority, 大山百合香, Hellberg, Various Artists, and Miles Davis
|
||||
/// *sixth*. An exact title match scores 100 whether the album is famous or
|
||||
/// was uploaded once. See [`rank_release_groups`].
|
||||
#[serde(default)]
|
||||
pub score: i32,
|
||||
/// Number of releases (pressings, editions, reissues) in this group.
|
||||
///
|
||||
/// **This is the popularity signal MusicBrainz does not otherwise have.**
|
||||
/// A work that has been pressed 135 times is the one people mean; a work
|
||||
/// with one release is somebody's upload. Measured on the same query:
|
||||
/// Miles Davis 135, every other score-100 hit exactly 1.
|
||||
#[serde(default)]
|
||||
pub count: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct Artist {
|
||||
#[serde(default)]
|
||||
pub id: String,
|
||||
#[serde(default)]
|
||||
pub name: String,
|
||||
/// `Person`, `Group`, `Orchestra`, `Choir`, `Character`, `Other`.
|
||||
#[serde(rename = "type", default)]
|
||||
pub artist_type: Option<String>,
|
||||
#[serde(default)]
|
||||
pub country: Option<String>,
|
||||
#[serde(default)]
|
||||
pub disambiguation: Option<String>,
|
||||
#[serde(default)]
|
||||
pub tags: Vec<Tag>,
|
||||
#[serde(default)]
|
||||
pub genres: Vec<Tag>,
|
||||
}
|
||||
|
||||
/// One track on a medium.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct Track {
|
||||
#[serde(default)]
|
||||
pub id: String,
|
||||
/// The printed track number — **a string, not an integer**: vinyl uses
|
||||
/// `A1`/`B2` and classical releases use `1.i`. `position` is the reliable
|
||||
/// ordinal.
|
||||
#[serde(default)]
|
||||
pub number: Option<String>,
|
||||
#[serde(default)]
|
||||
pub position: u32,
|
||||
#[serde(default)]
|
||||
pub title: String,
|
||||
/// Milliseconds. Absent on tracks nobody has timed.
|
||||
#[serde(default)]
|
||||
pub length: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub recording: Option<Recording>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct Recording {
|
||||
#[serde(default)]
|
||||
pub id: String,
|
||||
#[serde(default)]
|
||||
pub title: String,
|
||||
#[serde(default)]
|
||||
pub length: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub isrcs: Vec<String>,
|
||||
}
|
||||
|
||||
/// One physical/logical medium (a disc) within a release.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct Medium {
|
||||
#[serde(default)]
|
||||
pub position: u32,
|
||||
#[serde(default)]
|
||||
pub format: Option<String>,
|
||||
#[serde(rename = "track-count", default)]
|
||||
pub track_count: u32,
|
||||
#[serde(default)]
|
||||
pub tracks: Vec<Track>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct LabelInfo {
|
||||
#[serde(rename = "catalog-number", default)]
|
||||
pub catalog_number: Option<String>,
|
||||
#[serde(default)]
|
||||
pub label: Option<Label>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct Label {
|
||||
#[serde(default)]
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// A release: one specific pressing/edition of a release-group.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Default)]
|
||||
pub struct Release {
|
||||
#[serde(default)]
|
||||
pub id: String,
|
||||
#[serde(default)]
|
||||
pub title: String,
|
||||
#[serde(default)]
|
||||
pub date: Option<String>,
|
||||
#[serde(default)]
|
||||
pub country: Option<String>,
|
||||
#[serde(default)]
|
||||
pub media: Vec<Medium>,
|
||||
#[serde(rename = "label-info", default)]
|
||||
pub label_info: Vec<LabelInfo>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct ReleaseGroupSearch {
|
||||
#[serde(rename = "release-groups", default)]
|
||||
release_groups: Vec<ReleaseGroup>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct ArtistSearch {
|
||||
#[serde(default)]
|
||||
artists: Vec<Artist>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct ReleaseBrowse {
|
||||
#[serde(default)]
|
||||
releases: Vec<Release>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Client
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
pub struct MusicBrainzClient {
|
||||
http: reqwest::Client,
|
||||
base: String,
|
||||
budget: Arc<RateBudget>,
|
||||
cache: meta_feeder_sdk::cache::MidhashCache,
|
||||
}
|
||||
|
||||
impl MusicBrainzClient {
|
||||
pub fn new(budget: Arc<RateBudget>, cache: meta_feeder_sdk::cache::MidhashCache) -> Self {
|
||||
Self::with_base(MB_API_BASE.to_string(), budget, cache)
|
||||
}
|
||||
|
||||
/// Test constructor: point the client at a wiremock server.
|
||||
pub fn with_base(
|
||||
base: String,
|
||||
budget: Arc<RateBudget>,
|
||||
cache: meta_feeder_sdk::cache::MidhashCache,
|
||||
) -> Self {
|
||||
Self {
|
||||
http: reqwest::Client::builder()
|
||||
.user_agent(USER_AGENT)
|
||||
.timeout(Duration::from_secs(MB_TIMEOUT_SECS))
|
||||
.build()
|
||||
.expect("build musicbrainz http client"),
|
||||
base: base.trim_end_matches('/').to_string(),
|
||||
budget,
|
||||
cache,
|
||||
}
|
||||
}
|
||||
|
||||
/// One budget-gated GET returning the raw body.
|
||||
///
|
||||
/// Returns `None` on **every** failure path — deadline, transport, non-2xx.
|
||||
/// A feeder degrades to fewer results; it never propagates an upstream
|
||||
/// hiccup as a hard error, because a hard error empties the whole row
|
||||
/// rather than shortening it.
|
||||
async fn get_text(&self, path_and_query: &str, deadline: Duration) -> Option<String> {
|
||||
if self.budget.acquire(deadline).await == Lease::DeadlineExceeded {
|
||||
debug!(target: "meta-music", path = %path_and_query, "musicbrainz budget deadline; degrading");
|
||||
return None;
|
||||
}
|
||||
let url = format!("{}/{}", self.base, path_and_query.trim_start_matches('/'));
|
||||
let resp = match self.http.get(&url).send().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
warn!(target: "meta-music", %url, error = %e, "musicbrainz request failed");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
// MusicBrainz answers its throttle with 503 + Retry-After, not 429.
|
||||
// Reading this as a generic server error is how a feeder walks straight
|
||||
// into a block: it would keep hammering at full rate.
|
||||
if resp.status() == reqwest::StatusCode::SERVICE_UNAVAILABLE
|
||||
|| resp.status() == reqwest::StatusCode::TOO_MANY_REQUESTS
|
||||
{
|
||||
let retry = resp
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(MB_RATE_LIMIT_DEFAULT_RETRY_SECS)
|
||||
// ⚠ **Floor it.** `unwrap_or` only covers a *missing* or
|
||||
// unparseable header — and MusicBrainz sends a literal
|
||||
// `Retry-After: 0` under load, which parses fine and pauses the
|
||||
// budget for zero seconds. The client then retries immediately,
|
||||
// which is precisely how a throttled client stays throttled:
|
||||
// observed live as an unbroken run of "musicbrainz throttled;
|
||||
// pausing budget retry_after_s=0" while a plain curl to the
|
||||
// same URL answered 200.
|
||||
.max(MB_RATE_LIMIT_DEFAULT_RETRY_SECS);
|
||||
warn!(target: "meta-music", %url, retry_after_s = retry, "musicbrainz throttled; pausing budget");
|
||||
self.budget.note_throttled(Duration::from_secs(retry));
|
||||
return None;
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
warn!(target: "meta-music", %url, status = %resp.status(), "musicbrainz non-2xx");
|
||||
return None;
|
||||
}
|
||||
resp.text().await.ok()
|
||||
}
|
||||
|
||||
/// Search release-groups by free text. Cached permanently under the
|
||||
/// normalised query — a search result set is a ranking over immutable
|
||||
/// entities, and a stale ranking is a far cheaper error than an extra call
|
||||
/// against a 1/s budget.
|
||||
pub async fn search_release_groups(
|
||||
&self,
|
||||
text: &str,
|
||||
limit: usize,
|
||||
deadline: Duration,
|
||||
) -> Vec<ReleaseGroup> {
|
||||
let key = meta_feeder_sdk::cache::search_key("release-group", text);
|
||||
if let Ok(Some(hit)) = self.cache.get_mb_search(&key) {
|
||||
if let Ok(v) = serde_json::from_str::<Vec<ReleaseGroup>>(&hit) {
|
||||
return v.into_iter().take(limit).collect();
|
||||
}
|
||||
}
|
||||
// ⚠ Ask MusicBrainz for MORE than the caller wants.
|
||||
//
|
||||
// The re-ranking below is what surfaces the famous work, and it can
|
||||
// only rank what came back. On the measured `kind of blue` query the
|
||||
// Miles Davis release-group was **sixth** in MusicBrainz's own order —
|
||||
// so requesting the caller's `limit` of, say, 5 would have MusicBrainz
|
||||
// truncate him away before the ranking ever sees him, and no amount of
|
||||
// local sorting recovers that. One page is one request either way.
|
||||
let fetch = (limit * 4).clamp(MB_PAGE_SIZE, 100);
|
||||
let q = format!(
|
||||
"release-group?query={}&fmt=json&limit={}",
|
||||
urlencode(text),
|
||||
fetch
|
||||
);
|
||||
let Some(body) = self.get_text(&q, deadline).await else {
|
||||
return Vec::new();
|
||||
};
|
||||
let parsed: ReleaseGroupSearch = serde_json::from_str(&body).unwrap_or_default();
|
||||
// Rank BEFORE caching and before truncating: the cache stores the
|
||||
// ranked list, and truncating an unranked list to `limit` would throw
|
||||
// away the very hit the ranking exists to surface (Miles Davis was
|
||||
// sixth on the measured query — a `limit` of 5 would have dropped him).
|
||||
let out = rank_release_groups(parsed.release_groups);
|
||||
if !out.is_empty() {
|
||||
if let Ok(json) = serde_json::to_string(&out.iter().collect::<Vec<_>>()) {
|
||||
let _ = self.cache.put_mb_search(&key, &json);
|
||||
}
|
||||
}
|
||||
out.into_iter().take(limit).collect()
|
||||
}
|
||||
|
||||
/// Browse release-groups by **genre tag**.
|
||||
///
|
||||
/// ⚠ **This is why a genre row is a MusicBrainz search and not a
|
||||
/// ListenBrainz feed.** ListenBrainz has no genre parameter, so a first cut
|
||||
/// answered `genres:jazz` with the plain popularity feed and filtered the
|
||||
/// cards afterwards. That cannot work: a browse card carries no genres (they
|
||||
/// are resolved on the detail path), so the filter matched nothing — and
|
||||
/// falling back to the *unfiltered* feed was worse, because both the gateway
|
||||
/// and meta-search re-apply `record_matches` with `genres:jazz` and drop
|
||||
/// every card that does not carry it. Every genre row was empty either way.
|
||||
///
|
||||
/// MusicBrainz's Lucene index does have `tag:`, and a tag search returns
|
||||
/// cards whose `tags` then survive the re-filter. One call, real genres.
|
||||
pub async fn release_groups_by_tag(
|
||||
&self,
|
||||
tag: &str,
|
||||
limit: usize,
|
||||
deadline: Duration,
|
||||
) -> Vec<ReleaseGroup> {
|
||||
let key = meta_feeder_sdk::cache::search_key("rg-by-tag", tag);
|
||||
if let Ok(Some(hit)) = self.cache.get_mb_search(&key) {
|
||||
if let Ok(v) = serde_json::from_str::<Vec<ReleaseGroup>>(&hit) {
|
||||
return v.into_iter().take(limit).collect();
|
||||
}
|
||||
}
|
||||
// `primarytype:Album` keeps a genre row to full-length records; a page
|
||||
// of singles is not what "browse Jazz" means.
|
||||
let lucene = format!("tag:{tag} AND primarytype:Album");
|
||||
let q = format!(
|
||||
"release-group?query={}&fmt=json&limit={}",
|
||||
urlencode(&lucene),
|
||||
(limit * 3).clamp(MB_PAGE_SIZE, 100)
|
||||
);
|
||||
let Some(body) = self.get_text(&q, deadline).await else {
|
||||
return Vec::new();
|
||||
};
|
||||
let parsed: ReleaseGroupSearch = serde_json::from_str(&body).unwrap_or_default();
|
||||
let out = rank_release_groups(parsed.release_groups);
|
||||
if !out.is_empty() {
|
||||
if let Ok(json) = serde_json::to_string(&out.iter().collect::<Vec<_>>()) {
|
||||
let _ = self.cache.put_mb_search(&key, &json);
|
||||
}
|
||||
}
|
||||
out.into_iter().take(limit).collect()
|
||||
}
|
||||
|
||||
/// Search artists by free text.
|
||||
pub async fn search_artists(
|
||||
&self,
|
||||
text: &str,
|
||||
limit: usize,
|
||||
deadline: Duration,
|
||||
) -> Vec<Artist> {
|
||||
let key = meta_feeder_sdk::cache::search_key("artist", text);
|
||||
if let Ok(Some(hit)) = self.cache.get_mb_search(&key) {
|
||||
if let Ok(v) = serde_json::from_str::<Vec<Artist>>(&hit) {
|
||||
return v.into_iter().take(limit).collect();
|
||||
}
|
||||
}
|
||||
let q = format!(
|
||||
"artist?query={}&fmt=json&limit={}",
|
||||
urlencode(text),
|
||||
limit.clamp(1, MB_PAGE_SIZE * 4)
|
||||
);
|
||||
let Some(body) = self.get_text(&q, deadline).await else {
|
||||
return Vec::new();
|
||||
};
|
||||
let parsed: ArtistSearch = serde_json::from_str(&body).unwrap_or_default();
|
||||
let out = parsed.artists;
|
||||
if !out.is_empty() {
|
||||
if let Ok(json) = serde_json::to_string(&out.iter().collect::<Vec<_>>()) {
|
||||
let _ = self.cache.put_mb_search(&key, &json);
|
||||
}
|
||||
}
|
||||
out.into_iter().take(limit).collect()
|
||||
}
|
||||
|
||||
/// One release-group by MBID, with genres and artist credits.
|
||||
pub async fn release_group(&self, mbid: &str, deadline: Duration) -> Option<ReleaseGroup> {
|
||||
if let Ok(Some(hit)) = self.cache.get_mb_release_group(mbid) {
|
||||
if let Ok(v) = serde_json::from_str::<ReleaseGroup>(&hit) {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
let q = format!("release-group/{}?fmt=json&inc=artist-credits+genres", urlencode(mbid));
|
||||
let body = self.get_text(&q, deadline).await?;
|
||||
let rg: ReleaseGroup = serde_json::from_str(&body).ok()?;
|
||||
if rg.id.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let _ = self.cache.put_mb_release_group(mbid, &body);
|
||||
Some(rg)
|
||||
}
|
||||
|
||||
/// One artist by MBID, with genres.
|
||||
pub async fn artist(&self, mbid: &str, deadline: Duration) -> Option<Artist> {
|
||||
if let Ok(Some(hit)) = self.cache.get_mb_artist(mbid) {
|
||||
if let Ok(v) = serde_json::from_str::<Artist>(&hit) {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
let q = format!("artist/{}?fmt=json&inc=genres", urlencode(mbid));
|
||||
let body = self.get_text(&q, deadline).await?;
|
||||
let a: Artist = serde_json::from_str(&body).ok()?;
|
||||
if a.id.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let _ = self.cache.put_mb_artist(mbid, &body);
|
||||
Some(a)
|
||||
}
|
||||
|
||||
/// An artist's **YouTube channel relations**, in the two flavours
|
||||
/// MusicBrainz distinguishes: `("youtube music", url)` and `("youtube", url)`.
|
||||
///
|
||||
/// # Why this is an artist call and not a recording one
|
||||
///
|
||||
/// Measured, not assumed (study §4.1): two "Get Lucky" recordings and the
|
||||
/// *Random Access Memories* release-group returned **zero** YouTube
|
||||
/// relations between them. There is no per-track and no per-album shortcut
|
||||
/// — the links live at artist level, and only there. One lookup per artist,
|
||||
/// permanent, free, no key.
|
||||
///
|
||||
/// Returns the raw relation list; the caller ranks it. `None` is
|
||||
/// **"the lookup did not answer"**, which the caller must not confuse with
|
||||
/// "this artist has no channel" — see [`Self::get_text`], which reports a
|
||||
/// MusicBrainz `503` the same way it reports an empty result.
|
||||
/// Every name this artist might be searched under: the primary `name`, the
|
||||
/// `sort-name`, and all aliases.
|
||||
///
|
||||
/// # Why the credited string is not enough
|
||||
///
|
||||
/// MusicBrainz credits works in the artist's own script, and the name match
|
||||
/// against YouTube's artist index needs the one people actually type. The
|
||||
/// study's measured case: *Final Fantasy VII* is credited to `植松伸夫`,
|
||||
/// which finds no exact-name channel, even though `Nobuo Uematsu` resolves
|
||||
/// fine (§7.2.1). Without the aliases the channel key is simply unavailable
|
||||
/// for that artist, and every track then fails to reach EXACT.
|
||||
pub async fn artist_names(&self, mbid: &str, deadline: Duration) -> Vec<String> {
|
||||
let q = format!("artist/{}?fmt=json&inc=aliases", urlencode(mbid));
|
||||
let Some(body) = self.get_text(&q, deadline).await else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Ok(v) = serde_json::from_str::<serde_json::Value>(&body) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
let mut push = |s: Option<&str>| {
|
||||
if let Some(s) = s.map(str::trim).filter(|s| !s.is_empty()) {
|
||||
if !out.iter().any(|e: &String| e.eq_ignore_ascii_case(s)) {
|
||||
out.push(s.to_string());
|
||||
}
|
||||
}
|
||||
};
|
||||
push(v.get("name").and_then(|n| n.as_str()));
|
||||
push(v.get("sort-name").and_then(|n| n.as_str()));
|
||||
if let Some(aliases) = v.get("aliases").and_then(|a| a.as_array()) {
|
||||
for a in aliases {
|
||||
push(a.get("name").and_then(|n| n.as_str()));
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub async fn artist_url_relations(
|
||||
&self,
|
||||
mbid: &str,
|
||||
deadline: Duration,
|
||||
) -> Option<Vec<(String, String)>> {
|
||||
let q = format!("artist/{}?fmt=json&inc=url-rels", urlencode(mbid));
|
||||
let body = self.get_text(&q, deadline).await?;
|
||||
let v: serde_json::Value = serde_json::from_str(&body).ok()?;
|
||||
let rels = v.get("relations")?.as_array()?;
|
||||
Some(
|
||||
rels.iter()
|
||||
.filter_map(|r| {
|
||||
let ty = r.get("type")?.as_str()?.to_string();
|
||||
let url = r.get("url")?.get("resource")?.as_str()?.to_string();
|
||||
Some((ty, url))
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
|
||||
/// The canonical track list for a release-group.
|
||||
///
|
||||
/// **Why this is one call and a season walk is not.** MusicBrainz returns
|
||||
/// every medium and every track of a release in a single response, so an
|
||||
/// album detail page needs exactly one request here plus one retrieval
|
||||
/// query for availability. There is no per-disc fan-out, no walk, no TTL
|
||||
/// that could hide a track — the shape of the data removes the problem
|
||||
/// that `meta-watch`'s `season_walk.rs` exists to manage.
|
||||
///
|
||||
/// Which pressing counts as canonical: the **first release returned that
|
||||
/// actually carries tracks**. MusicBrainz orders a release-group's
|
||||
/// releases by date, so this is the earliest complete one — the edition
|
||||
/// whose track list people mean by "the album". Later remasters add bonus
|
||||
/// tracks, which is exactly what should not define the canonical list.
|
||||
pub async fn tracklist(&self, release_group_mbid: &str, deadline: Duration) -> Option<Release> {
|
||||
if let Ok(Some(hit)) = self.cache.get_mb_tracklist(release_group_mbid) {
|
||||
if let Ok(v) = serde_json::from_str::<Release>(&hit) {
|
||||
return Some(v);
|
||||
}
|
||||
}
|
||||
let q = format!(
|
||||
"release?release-group={}&fmt=json&inc=recordings+artist-credits+isrcs+labels&limit=5",
|
||||
urlencode(release_group_mbid)
|
||||
);
|
||||
let body = self.get_text(&q, deadline).await?;
|
||||
let parsed: ReleaseBrowse = serde_json::from_str(&body).unwrap_or_default();
|
||||
let chosen = parsed
|
||||
.releases
|
||||
.into_iter()
|
||||
.find(|r| r.media.iter().any(|m| !m.tracks.is_empty()))?;
|
||||
if let Ok(json) = serde_json::to_string(&chosen) {
|
||||
let _ = self.cache.put_mb_tracklist(release_group_mbid, &json);
|
||||
}
|
||||
Some(chosen)
|
||||
}
|
||||
|
||||
/// A browse of an artist's release-groups — their discography.
|
||||
pub async fn release_groups_by_artist(
|
||||
&self,
|
||||
artist_mbid: &str,
|
||||
limit: usize,
|
||||
deadline: Duration,
|
||||
) -> Vec<ReleaseGroup> {
|
||||
let key = meta_feeder_sdk::cache::search_key("rg-by-artist", artist_mbid);
|
||||
if let Ok(Some(hit)) = self.cache.get_mb_search(&key) {
|
||||
if let Ok(v) = serde_json::from_str::<Vec<ReleaseGroup>>(&hit) {
|
||||
return v.into_iter().take(limit).collect();
|
||||
}
|
||||
}
|
||||
let q = format!(
|
||||
"release-group?artist={}&fmt=json&limit={}",
|
||||
urlencode(artist_mbid),
|
||||
limit.clamp(1, 100)
|
||||
);
|
||||
let Some(body) = self.get_text(&q, deadline).await else {
|
||||
return Vec::new();
|
||||
};
|
||||
let parsed: ReleaseGroupSearch = serde_json::from_str(&body).unwrap_or_default();
|
||||
let out = parsed.release_groups;
|
||||
if !out.is_empty() {
|
||||
if let Ok(json) = serde_json::to_string(&out.iter().collect::<Vec<_>>()) {
|
||||
let _ = self.cache.put_mb_search(&key, &json);
|
||||
}
|
||||
}
|
||||
out.into_iter().take(limit).collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// Percent-encode a query-string value. Deliberately conservative — the Lucene
|
||||
/// syntax MusicBrainz accepts uses `:`/`(`/`)`/`"` meaningfully, so anything
|
||||
/// outside the unreserved set is escaped rather than passed through.
|
||||
pub(crate) fn urlencode(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len() * 2);
|
||||
for b in s.bytes() {
|
||||
match b {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => {
|
||||
out.push(b as char)
|
||||
}
|
||||
b' ' => out.push_str("%20"),
|
||||
_ => out.push_str(&format!("%{b:02X}")),
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Re-rank release-group search hits: relevance band first, then how many
|
||||
/// times the work has actually been released.
|
||||
///
|
||||
/// **Why this is not MusicBrainz's own order.** Their `score` is a Lucene
|
||||
/// relevance figure that saturates at 100 for any exact title match, so a
|
||||
/// search for a famous album returns it *behind* every obscure record that
|
||||
/// happens to share its name. Measured live on `query=kind of blue`:
|
||||
///
|
||||
/// | # | score | releases | artist |
|
||||
/// |---|-------|----------|--------|
|
||||
/// | 0 | 100 | 1 | Swiss Blues Authority |
|
||||
/// | 1 | 100 | 1 | TQ |
|
||||
/// | 2 | 100 | 1 | 大山百合香 |
|
||||
/// | … | … | … | … |
|
||||
/// | 5 | 100 | **135** | **Miles Davis** |
|
||||
///
|
||||
/// TMDB solves this with a popularity field; MusicBrainz has none. The release
|
||||
/// count is the closest thing it does carry, it comes free in the same
|
||||
/// response, and it is decisive here.
|
||||
///
|
||||
/// **Score stays the primary key on purpose.** Sorting by release count alone
|
||||
/// would let a prolific artist's loosely-matching album outrank an exact hit.
|
||||
/// The count only breaks ties *within* a relevance band — which is exactly the
|
||||
/// case the ranking fails at, since MusicBrainz's bands are coarse (100, 89,
|
||||
/// 80, 76, …) and an exact-match band is where the pile-up happens.
|
||||
pub fn rank_release_groups(mut hits: Vec<ReleaseGroup>) -> Vec<ReleaseGroup> {
|
||||
hits.sort_by(|a, b| {
|
||||
b.score
|
||||
.cmp(&a.score)
|
||||
.then_with(|| b.count.cmp(&a.count))
|
||||
// Stable, deterministic final tiebreak so a cached ranking and a
|
||||
// fresh one cannot disagree.
|
||||
.then_with(|| a.id.cmp(&b.id))
|
||||
});
|
||||
hits
|
||||
}
|
||||
|
||||
/// Normalise a MusicBrainz variable-precision date to what `releasedate`
|
||||
/// expects.
|
||||
///
|
||||
/// MusicBrainz emits `YYYY`, `YYYY-MM` or `YYYY-MM-DD` depending on what is
|
||||
/// known. `METADATA_KEYS.md`'s `iso-date` format is `YYYY-MM-DD`, so a partial
|
||||
/// date is completed with `-01` rather than dropped — losing the year entirely
|
||||
/// because the day is unknown would be the worse error, and every consumer
|
||||
/// that renders this only shows the year.
|
||||
pub fn normalise_date(raw: &str) -> Option<String> {
|
||||
// ⚠ Validate the shape, do not infer it from the length. A first cut
|
||||
// matched on `len()` alone, which turned the seven-character string
|
||||
// "garbage" into the `releasedate` "garbage-01" — a malformed value
|
||||
// written to the shared hash, where nothing downstream would ever question
|
||||
// it. Every segment is checked for digits and width.
|
||||
let digits = |s: &str, n: usize| s.len() == n && s.chars().all(|c| c.is_ascii_digit());
|
||||
let parts: Vec<&str> = raw.trim().split('-').collect();
|
||||
match parts.as_slice() {
|
||||
[y] if digits(y, 4) => Some(format!("{y}-01-01")),
|
||||
[y, m] if digits(y, 4) && digits(m, 2) => Some(format!("{y}-{m}-01")),
|
||||
[y, m, d] if digits(y, 4) && digits(m, 2) && digits(d, 2) => Some(format!("{y}-{m}-{d}")),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The year from a MusicBrainz variable-precision date.
|
||||
pub fn year_of(raw: &str) -> Option<u16> {
|
||||
raw.trim().get(0..4)?.parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn credit_rendering_honours_join_phrases() {
|
||||
let credits = vec![
|
||||
ArtistCredit {
|
||||
name: "Miles Davis".into(),
|
||||
joinphrase: " feat. ".into(),
|
||||
artist: Some(CreditedArtist {
|
||||
id: "561d854a-6a28-4aa7-8c99-323e6ce46c2a".into(),
|
||||
name: "Miles Davis".into(),
|
||||
}),
|
||||
},
|
||||
ArtistCredit {
|
||||
name: "John Coltrane".into(),
|
||||
joinphrase: String::new(),
|
||||
artist: Some(CreditedArtist {
|
||||
id: "b625448e-bf4a-41c3-a421-72ad46cdb831".into(),
|
||||
name: "John Coltrane".into(),
|
||||
}),
|
||||
},
|
||||
];
|
||||
assert_eq!(render_credit(&credits), "Miles Davis feat. John Coltrane");
|
||||
assert_eq!(
|
||||
primary_artist(&credits).map(|a| a.name.as_str()),
|
||||
Some("Miles Davis")
|
||||
);
|
||||
}
|
||||
|
||||
/// A credit entry with no resolved artist must not become the primary —
|
||||
/// the card would be filed under an artist MBID that does not exist.
|
||||
#[test]
|
||||
fn primary_artist_skips_credits_with_no_mbid() {
|
||||
let credits = vec![
|
||||
ArtistCredit {
|
||||
name: "Various Artists".into(),
|
||||
joinphrase: String::new(),
|
||||
artist: None,
|
||||
},
|
||||
ArtistCredit {
|
||||
name: "Real Artist".into(),
|
||||
joinphrase: String::new(),
|
||||
artist: Some(CreditedArtist {
|
||||
id: "abc".into(),
|
||||
name: "Real Artist".into(),
|
||||
}),
|
||||
},
|
||||
];
|
||||
assert_eq!(primary_artist(&credits).map(|a| a.id.as_str()), Some("abc"));
|
||||
}
|
||||
|
||||
/// Variable-precision dates are the norm, not the exception — a
|
||||
/// year-only release date must still produce a usable `releasedate`.
|
||||
#[test]
|
||||
fn dates_are_completed_not_dropped() {
|
||||
assert_eq!(normalise_date("1959").as_deref(), Some("1959-01-01"));
|
||||
assert_eq!(normalise_date("1959-08").as_deref(), Some("1959-08-01"));
|
||||
assert_eq!(normalise_date("1959-08-17").as_deref(), Some("1959-08-17"));
|
||||
assert_eq!(normalise_date(""), None);
|
||||
// ⚠ Regression guard: "garbage" is seven characters, exactly the width
|
||||
// of `YYYY-MM`. A length-only check wrote "garbage-01" into
|
||||
// `releasedate` on the shared hash.
|
||||
assert_eq!(normalise_date("garbage"), None);
|
||||
assert_eq!(normalise_date("19xx"), None);
|
||||
assert_eq!(normalise_date("1959-ab"), None);
|
||||
assert_eq!(normalise_date("1959-08-1x"), None);
|
||||
assert_eq!(normalise_date("1959-8-17"), None, "segments are fixed-width");
|
||||
assert_eq!(year_of("1959-08-17"), Some(1959));
|
||||
assert_eq!(year_of("19"), None);
|
||||
}
|
||||
|
||||
fn rg(id: &str, score: i32, count: u32) -> ReleaseGroup {
|
||||
ReleaseGroup {
|
||||
id: id.to_string(),
|
||||
title: "Kind of Blue".to_string(),
|
||||
score,
|
||||
count,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// ⚠ The measured failure this exists for: on `query=kind of blue`,
|
||||
/// MusicBrainz returned six release-groups all scoring exactly 100, with
|
||||
/// **Miles Davis sixth** behind five one-release records that happen to
|
||||
/// share the title. Release count is the only discriminator in the payload.
|
||||
#[test]
|
||||
fn release_count_breaks_a_saturated_score_tie() {
|
||||
let hits = vec![
|
||||
rg("swiss", 100, 1),
|
||||
rg("tq", 100, 1),
|
||||
rg("oyama", 100, 1),
|
||||
rg("hellberg", 100, 1),
|
||||
rg("various", 100, 1),
|
||||
rg("miles", 100, 135),
|
||||
];
|
||||
let ranked = rank_release_groups(hits);
|
||||
assert_eq!(ranked[0].id, "miles", "the 135-release work must lead");
|
||||
}
|
||||
|
||||
/// …but relevance still wins over popularity across bands, or a prolific
|
||||
/// artist's loosely-matching album would outrank an exact hit.
|
||||
#[test]
|
||||
fn score_outranks_release_count_across_bands() {
|
||||
let ranked = rank_release_groups(vec![rg("loose", 89, 500), rg("exact", 100, 2)]);
|
||||
assert_eq!(ranked[0].id, "exact");
|
||||
}
|
||||
|
||||
/// A cached ranking and a fresh one must not disagree, so ties are broken
|
||||
/// deterministically rather than left to sort stability.
|
||||
#[test]
|
||||
fn ranking_is_deterministic_for_identical_hits() {
|
||||
let a = rank_release_groups(vec![rg("bbb", 100, 1), rg("aaa", 100, 1)]);
|
||||
let b = rank_release_groups(vec![rg("aaa", 100, 1), rg("bbb", 100, 1)]);
|
||||
assert_eq!(
|
||||
a.iter().map(|r| r.id.as_str()).collect::<Vec<_>>(),
|
||||
b.iter().map(|r| r.id.as_str()).collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
/// The Lucene metacharacters MusicBrainz treats as syntax must be escaped,
|
||||
/// or a punctuated title (`Re:ZERO`, `!!!`) is read as a field query.
|
||||
#[test]
|
||||
fn urlencode_escapes_lucene_metacharacters() {
|
||||
assert_eq!(urlencode("Re:ZERO"), "Re%3AZERO");
|
||||
assert_eq!(urlencode("kind of blue"), "kind%20of%20blue");
|
||||
assert_eq!(urlencode("!!!"), "%21%21%21");
|
||||
assert_eq!(urlencode("safe-Chars_1.0~"), "safe-Chars_1.0~");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
//! Constants for the music feeder.
|
||||
//!
|
||||
//! The music-domain counterpart of the card feeder's `consts.rs`. Where a knob
|
||||
//! has the same job as one over there, it keeps the same name and the same
|
||||
//! reasoning so the two feeders stay readable side by side.
|
||||
|
||||
/// User-Agent for outbound HTTP.
|
||||
///
|
||||
/// ⚠ **Not cosmetic for MusicBrainz.** Their API rejects requests with a
|
||||
/// generic or absent User-Agent, and the rule is that it must identify the
|
||||
/// application and carry contact information. A 403 here looks exactly like a
|
||||
/// rate-limit and is easy to misdiagnose, so keep this descriptive.
|
||||
pub const USER_AGENT: &str = concat!(
|
||||
"meta-music/",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
" ( https://github.com/worph/meta-feeder-music )"
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MusicBrainz
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// MusicBrainz web-service v2 base.
|
||||
pub const MB_API_BASE: &str = "https://musicbrainz.org/ws/2";
|
||||
|
||||
/// Sustained MusicBrainz request rate.
|
||||
///
|
||||
/// **One per second is the service's published ceiling, not a suggestion** —
|
||||
/// exceeding it earns a block, and the block is applied per source IP, so one
|
||||
/// misbehaving feeder takes out every peer behind the same NAT. Unlike TMDB's
|
||||
/// (removed) limit there is no headroom to trade for latency here.
|
||||
pub const MB_RATE_PER_SEC: f64 = 1.0;
|
||||
|
||||
/// MusicBrainz burst ceiling. Deliberately tiny: the rate above is a hard
|
||||
/// ceiling, so a burst only borrows against the next second.
|
||||
pub const MB_BURST: f64 = 2.0;
|
||||
|
||||
/// Request timeout for a MusicBrainz call (seconds).
|
||||
pub const MB_TIMEOUT_SECS: u64 = 20;
|
||||
|
||||
/// Default `Retry-After` applied when MusicBrainz returns 503 with no (or an
|
||||
/// unparseable) header. Their throttle answers 503, not 429.
|
||||
pub const MB_RATE_LIMIT_DEFAULT_RETRY_SECS: u64 = 2;
|
||||
|
||||
/// Budget-permit deadline for a query somebody is waiting on.
|
||||
///
|
||||
/// Far shorter than the card feeder's 3000 s TMDB equivalent, because the
|
||||
/// budget here is 1/s rather than 20/s: a deep queue means the answer is
|
||||
/// minutes away, and no search box should hold a connection open that long.
|
||||
pub const MB_WAIT_DEADLINE_SECS: u64 = 30;
|
||||
|
||||
/// Budget-permit deadline for a **browse row**, deliberately far shorter than
|
||||
/// [`MB_WAIT_DEADLINE_SECS`].
|
||||
///
|
||||
/// Same reasoning as the card feeder's `TMDB_DISCOVERY_WAIT_DEADLINE_SECS`:
|
||||
/// nobody asked for *this* row in particular, it refreshes on a timer, and the
|
||||
/// consumer keeps the previously-filled row on an empty pass (meta-listen's
|
||||
/// sticky per-row merge). Waiting out a queue to eventually produce what a
|
||||
/// fast bail produces now just pins a search stream open.
|
||||
pub const MB_DISCOVERY_WAIT_DEADLINE_SECS: u64 = 5;
|
||||
|
||||
/// Results MusicBrainz returns per search page (its own default and maximum
|
||||
/// useful page size for our purposes).
|
||||
pub const MB_PAGE_SIZE: usize = 25;
|
||||
|
||||
/// Hard ceiling on MusicBrainz pages walked for one row, whatever the
|
||||
/// requested card count. Bounds a misconfigured `card_discovery_n` to a
|
||||
/// predictable per-row cost against a 1/s budget.
|
||||
pub const MB_MAX_PAGES: u32 = 3;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cover Art Archive
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Cover Art Archive base. Keyed by MusicBrainz entity id.
|
||||
pub const CAA_BASE: &str = "https://coverartarchive.org";
|
||||
|
||||
/// Cover size segment. `front-500` is the grid sweet spot — roughly 60 KB,
|
||||
/// against `front`'s frequently multi-MB original scan.
|
||||
pub const CAA_FRONT_SIZE: &str = "front-500";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ListenBrainz
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// ListenBrainz API base. No credential needed for the sitewide/browse
|
||||
/// endpoints this feeder reads.
|
||||
pub const LB_API_BASE: &str = "https://api.listenbrainz.org/1";
|
||||
|
||||
/// Sustained ListenBrainz request rate. Their limit is generous and
|
||||
/// header-advertised; this is a courtesy ceiling, not a published one.
|
||||
pub const LB_RATE_PER_SEC: f64 = 4.0;
|
||||
|
||||
/// ListenBrainz burst ceiling.
|
||||
pub const LB_BURST: f64 = 8.0;
|
||||
|
||||
/// Request timeout for a ListenBrainz call (seconds).
|
||||
pub const LB_TIMEOUT_SECS: u64 = 20;
|
||||
|
||||
/// Statistics range passed to the sitewide stats endpoints.
|
||||
///
|
||||
/// ⚠ The parameter is `range`. It was `stats_range` until a documented
|
||||
/// breaking change; a request using the old name is answered with a 400 that
|
||||
/// reads like a bad endpoint.
|
||||
pub const LB_STATS_RANGE: &str = "month";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result sizing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Default number of cards a free-text search returns.
|
||||
///
|
||||
/// Matches the card feeder's `DEFAULT_CARD_TOP_N` and costs the same: at most
|
||||
/// one cached MusicBrainz search. The retrieval budget is spent later, on the
|
||||
/// single card the user clicks.
|
||||
pub const DEFAULT_CARD_TOP_N: usize = 10;
|
||||
|
||||
/// Default number of cards a keyword-less browse row returns.
|
||||
pub const DEFAULT_CARD_DISCOVERY_N: usize = 20;
|
||||
|
||||
/// Over-fetch factor applied when walking MusicBrainz pages for a browse row.
|
||||
///
|
||||
/// **Why a row asks for more works than it needs.** The quality gate hides any
|
||||
/// album card with no cover art, and — unlike the card feeder, where TMDB's
|
||||
/// own response says whether a cover exists — nothing in a MusicBrainz
|
||||
/// response reveals whether the Cover Art Archive holds a front image for a
|
||||
/// release-group. The only way to know is to fetch it, which the **gateway
|
||||
/// core** does (it upgrades the `cover` locator to a seeded cid), long after
|
||||
/// this feeder has returned.
|
||||
///
|
||||
/// So the drop is genuinely unpredictable here and the row would simply come
|
||||
/// up short. Over-fetching trades a bounded number of extra rows-worth of
|
||||
/// candidates for a full row. It is a mitigation, not a fix: a row can still
|
||||
/// be short, and the consumer's sticky per-row merge is what makes that
|
||||
/// acceptable.
|
||||
pub const DISCOVERY_OVERFETCH: usize = 2;
|
||||
|
||||
/// Defensive ceiling on a release's reported track count (guards a corrupt or
|
||||
/// absurd upstream value from sizing an allocation).
|
||||
pub const MAX_TRACK_COUNT: u32 = 500;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// youtube — the delegated-playback tier
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -0,0 +1,357 @@
|
||||
//! Keyword-less browse: ListenBrainz popularity and fresh-release feeds →
|
||||
//! [`Card`]s.
|
||||
//!
|
||||
//! The card tier's *browse* surface as opposed to its search surface. A client
|
||||
//! with a home board cannot ask "what should I show?" through a text query —
|
||||
//! there is no text. This branch answers a query whose intent is carried
|
||||
//! entirely by structured filters.
|
||||
//!
|
||||
//! ## Query dialect
|
||||
//!
|
||||
//! Deliberately identical to the card feeder's, so a client switches domains
|
||||
//! by swapping `contentKind:series` for `contentKind:album` and nothing else:
|
||||
//!
|
||||
//! - **mode** (one of) — `popular:true` → ListenBrainz sitewide top;
|
||||
//! `trending:true` → the same feed over a shorter window (ListenBrainz has no
|
||||
//! distinct trending endpoint, so trending approximates to recent
|
||||
//! popularity, exactly as the card feeder's anime path approximates trending
|
||||
//! with popularity); `fresh:true` → the fresh-releases feed.
|
||||
//! - **kind** (required) — `contentKind:album` → albums, `contentKind:artist`
|
||||
//! → artists. Also what routes the query here at all, alongside
|
||||
//! `fileType:card`.
|
||||
//!
|
||||
//! ⚠ There is deliberately **no `top_rated` mode**. ListenBrainz measures
|
||||
//! listens, not ratings, and MusicBrainz holds no rating at all — a
|
||||
//! `top_rated:true` row would have to be silently answered with popularity,
|
||||
//! which is exactly the kind of quiet substitution that makes a browse row
|
||||
//! impossible to reason about. An unsupported mode returns nothing and says so
|
||||
//! in the log.
|
||||
//!
|
||||
//! ## Why nothing here is cached
|
||||
//!
|
||||
//! A popularity list is **mutable by nature** — "popular this month" is the one
|
||||
//! thing that must not be frozen — and this crate's cache is a permanent-hit
|
||||
//! store sized for immutable MBID-keyed data. Persisting a chart there would
|
||||
//! pin the board to whatever it looked like on first boot. The layer above
|
||||
//! already absorbs the repeat cost: the gateway persists each emitted card to
|
||||
//! meta-core and its search-coverage gate then skips a repeated
|
||||
//! `(upstream, query)` for an hour, serving meta-core instead. Every call is
|
||||
//! budget-gated, so a burst cannot run away.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use meta_feeder_sdk::plugin::GatewayQuery;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::card::{Card, CardKind};
|
||||
use crate::consts::DISCOVERY_OVERFETCH;
|
||||
use crate::resolve::Resolver;
|
||||
|
||||
/// Days back the `fresh:true` row reaches.
|
||||
const FRESH_WINDOW_DAYS: u32 = 30;
|
||||
|
||||
/// The browse flavour a query asks for.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
pub enum BrowseMode {
|
||||
Popular,
|
||||
Trending,
|
||||
Fresh,
|
||||
}
|
||||
|
||||
impl BrowseMode {
|
||||
/// The query-filter key that selects this mode.
|
||||
///
|
||||
/// It is also echoed onto every emitted record — not here, but by
|
||||
/// [`Card::to_record`]'s trailing filter echo, which stamps back every
|
||||
/// filter the query carried that the card did not already set. That echo
|
||||
/// is load-bearing: both the gateway dispatcher and meta-search re-apply
|
||||
/// `record_matches` to each record, and a *missing* field fails the match,
|
||||
/// so an un-echoed `popular` would see every card in the row dropped at
|
||||
/// one of the two tiers.
|
||||
fn marker(self) -> &'static str {
|
||||
match self {
|
||||
BrowseMode::Popular => "popular",
|
||||
BrowseMode::Trending => "trending",
|
||||
BrowseMode::Fresh => "fresh",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// True when this is a keyword-less browse query this branch should answer.
|
||||
///
|
||||
/// Three conditions, each excluding a query that belongs to a *different*
|
||||
/// branch of `handle_query`: no free text (that is the principal search), no
|
||||
/// MBID filter (that is the direct by-id lookup a deep link uses), and a
|
||||
/// truthy mode marker. `contentKind` alone deliberately does **not** select
|
||||
/// this branch — a text query carries it too.
|
||||
pub fn is_browse_query(query: &GatewayQuery) -> bool {
|
||||
query.free_text.trim().is_empty()
|
||||
&& !query.filters.contains_key("mbReleaseGroupId")
|
||||
&& !query.filters.contains_key("mbArtistId")
|
||||
&& browse_mode(query).is_some()
|
||||
}
|
||||
|
||||
/// First truthy mode marker on the query, in precedence order.
|
||||
fn browse_mode(query: &GatewayQuery) -> Option<BrowseMode> {
|
||||
for mode in [BrowseMode::Fresh, BrowseMode::Trending, BrowseMode::Popular] {
|
||||
if filter_is_true(query, mode.marker()) {
|
||||
return Some(mode);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// True iff `query.filters[key]` carries a truthy value.
|
||||
fn filter_is_true(query: &GatewayQuery, key: &str) -> bool {
|
||||
query
|
||||
.filters
|
||||
.get(key)
|
||||
.is_some_and(|v| v.iter().any(|s| s.eq_ignore_ascii_case("true")))
|
||||
}
|
||||
|
||||
/// Map the query's `contentKind` filter to a card kind.
|
||||
pub fn browse_kind(query: &GatewayQuery) -> Option<CardKind> {
|
||||
let values = query.filters.get("contentKind")?;
|
||||
for v in values {
|
||||
match v.trim().to_ascii_lowercase().as_str() {
|
||||
"album" => return Some(CardKind::Album),
|
||||
"artist" => return Some(CardKind::Artist),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// The genre slug a row is filtered by, if any.
|
||||
///
|
||||
/// ⚠ A genre row is answered by a **MusicBrainz tag search**, not by filtering
|
||||
/// a ListenBrainz feed — see [`MusicBrainzClient::release_groups_by_tag`] for
|
||||
/// why the filtering approach cannot work at all.
|
||||
fn genre_slug(query: &GatewayQuery) -> Option<String> {
|
||||
query
|
||||
.filters
|
||||
.get("genres")
|
||||
.and_then(|v| v.first())
|
||||
.map(|s| s.trim().to_ascii_lowercase())
|
||||
.filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
/// A genre slug back to the tag MusicBrainz indexes it under. The query DSL
|
||||
/// cannot spell a name with spaces (space is its token separator), so the wire
|
||||
/// carries `hip-hop` and MusicBrainz wants `hip hop`.
|
||||
fn slug_to_tag(slug: &str) -> String {
|
||||
slug.replace('-', " ")
|
||||
}
|
||||
|
||||
/// Answer a browse query.
|
||||
pub async fn cards_for_browse(
|
||||
resolver: &Resolver,
|
||||
query: &GatewayQuery,
|
||||
n: usize,
|
||||
deadline: Duration,
|
||||
) -> Vec<Card> {
|
||||
let Some(mode) = browse_mode(query) else {
|
||||
return Vec::new();
|
||||
};
|
||||
let Some(kind) = browse_kind(query) else {
|
||||
warn!(
|
||||
target: "meta-music",
|
||||
content_kind = ?query.filters.get("contentKind"),
|
||||
"browse row has no album/artist contentKind; a card is a WORK — \
|
||||
use contentKind:album or contentKind:artist"
|
||||
);
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
// Over-fetch, because the cover gate's drop rate is not predictable here —
|
||||
// see `consts::DISCOVERY_OVERFETCH`.
|
||||
let want = n.saturating_mul(DISCOVERY_OVERFETCH).clamp(1, 100);
|
||||
|
||||
// A genre row is its own path: MusicBrainz tag search, not a filtered feed.
|
||||
if let (CardKind::Album, Some(slug)) = (kind, genre_slug(query)) {
|
||||
let tag = slug_to_tag(&slug);
|
||||
let mut cards = resolver.album_cards_for_tag(&tag, want, deadline).await;
|
||||
if cards.is_empty() {
|
||||
debug!(target: "meta-music", %tag, "genre row returned nothing");
|
||||
}
|
||||
cards.truncate(n);
|
||||
return cards;
|
||||
}
|
||||
|
||||
let mut cards: Vec<Card> = match (kind, mode) {
|
||||
(CardKind::Album, BrowseMode::Fresh) => resolver
|
||||
.lb
|
||||
.fresh_releases(FRESH_WINDOW_DAYS, deadline)
|
||||
.await
|
||||
.iter()
|
||||
.filter_map(|w| resolver.album_card_from_popular(w))
|
||||
.collect(),
|
||||
(CardKind::Album, _) => resolver
|
||||
.lb
|
||||
.popular_release_groups(want, deadline)
|
||||
.await
|
||||
.iter()
|
||||
.filter_map(|w| resolver.album_card_from_popular(w))
|
||||
.collect(),
|
||||
(CardKind::Artist, BrowseMode::Fresh) => {
|
||||
// "Fresh artists" is not a thing ListenBrainz reports, and inventing
|
||||
// it from the fresh-release feed would silently answer a different
|
||||
// question. Say so and return nothing.
|
||||
debug!(target: "meta-music", "fresh:true is not defined for contentKind:artist");
|
||||
Vec::new()
|
||||
}
|
||||
(CardKind::Artist, _) => {
|
||||
let popular = resolver.lb.popular_artists(want, deadline).await;
|
||||
resolver
|
||||
.artist_cards_from_popular(&popular, n, deadline)
|
||||
.await
|
||||
}
|
||||
};
|
||||
|
||||
cards.truncate(n);
|
||||
cards
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::card::{CardSource, Cover};
|
||||
|
||||
fn q(raw: &str, filters: &[(&str, &str)]) -> GatewayQuery {
|
||||
let mut q = GatewayQuery::from_free_text(raw);
|
||||
for (k, v) in filters {
|
||||
q.filters
|
||||
.entry(k.to_string())
|
||||
.or_default()
|
||||
.push(v.to_string());
|
||||
}
|
||||
q
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_keywordless_mode_query_is_a_browse_query() {
|
||||
assert!(is_browse_query(&q(
|
||||
"",
|
||||
&[("popular", "true"), ("contentKind", "album")]
|
||||
)));
|
||||
assert!(is_browse_query(&q(
|
||||
"",
|
||||
&[("fresh", "true"), ("contentKind", "album")]
|
||||
)));
|
||||
}
|
||||
|
||||
/// Free text means the principal search owns the query, mode marker or not.
|
||||
#[test]
|
||||
fn free_text_excludes_the_browse_branch() {
|
||||
assert!(!is_browse_query(&q(
|
||||
"miles davis",
|
||||
&[("popular", "true"), ("contentKind", "album")]
|
||||
)));
|
||||
}
|
||||
|
||||
/// An explicit id is the direct lookup, which is a third branch.
|
||||
#[test]
|
||||
fn an_id_filter_excludes_the_browse_branch() {
|
||||
assert!(!is_browse_query(&q(
|
||||
"",
|
||||
&[
|
||||
("popular", "true"),
|
||||
("contentKind", "album"),
|
||||
("mbReleaseGroupId", "rg-1")
|
||||
]
|
||||
)));
|
||||
assert!(!is_browse_query(&q(
|
||||
"",
|
||||
&[("popular", "true"), ("mbArtistId", "a-1")]
|
||||
)));
|
||||
}
|
||||
|
||||
/// `contentKind` alone must not select the browse branch — a text query
|
||||
/// carries it too.
|
||||
#[test]
|
||||
fn a_content_kind_alone_is_not_a_browse_query() {
|
||||
assert!(!is_browse_query(&q("", &[("contentKind", "album")])));
|
||||
}
|
||||
|
||||
/// The card feeder's vocabulary must NOT resolve here. A row written for
|
||||
/// video (`contentKind:series`) reaching the music feeder is a
|
||||
/// misconfiguration, and answering it with albums would hide that.
|
||||
#[test]
|
||||
fn video_content_kinds_do_not_resolve_to_a_music_kind() {
|
||||
assert_eq!(browse_kind(&q("", &[("contentKind", "series")])), None);
|
||||
assert_eq!(browse_kind(&q("", &[("contentKind", "movie")])), None);
|
||||
assert_eq!(
|
||||
browse_kind(&q("", &[("contentKind", "album")])),
|
||||
Some(CardKind::Album)
|
||||
);
|
||||
assert_eq!(
|
||||
browse_kind(&q("", &[("contentKind", "artist")])),
|
||||
Some(CardKind::Artist)
|
||||
);
|
||||
}
|
||||
|
||||
/// `top_rated` is deliberately unsupported — nothing upstream measures
|
||||
/// ratings, and quietly answering with popularity would make the row a lie.
|
||||
#[test]
|
||||
fn top_rated_is_not_a_mode() {
|
||||
assert!(!is_browse_query(&q(
|
||||
"",
|
||||
&[("top_rated", "true"), ("contentKind", "album")]
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mode_precedence_is_fresh_then_trending_then_popular() {
|
||||
let all = q(
|
||||
"",
|
||||
&[("popular", "true"), ("trending", "true"), ("fresh", "true")],
|
||||
);
|
||||
assert_eq!(browse_mode(&all), Some(BrowseMode::Fresh));
|
||||
let two = q("", &[("popular", "true"), ("trending", "true")]);
|
||||
assert_eq!(browse_mode(&two), Some(BrowseMode::Trending));
|
||||
}
|
||||
|
||||
fn card_with_genres(genres: &[&str]) -> Card {
|
||||
Card {
|
||||
source: CardSource::MusicBrainz,
|
||||
kind: CardKind::Album,
|
||||
mbid: "rg".into(),
|
||||
title: "T".into(),
|
||||
album_artist: None,
|
||||
artists: vec![],
|
||||
artist_mbid: None,
|
||||
cover: Some(Cover::ReleaseGroupFront {
|
||||
release_group_mbid: "rg".into(),
|
||||
}),
|
||||
disambiguation: None,
|
||||
genres: genres.iter().map(|s| s.to_string()).collect(),
|
||||
release_date: None,
|
||||
track_count: None,
|
||||
disc_count: None,
|
||||
label: None,
|
||||
catalog_number: None,
|
||||
primary_type: None,
|
||||
secondary_types: vec![],
|
||||
tracks: Vec::new(),
|
||||
streaming_links: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The query language cannot spell "Hip Hop" (space is its token
|
||||
/// separator), so the wire carries a slug and MusicBrainz wants the tag.
|
||||
#[test]
|
||||
fn a_genre_slug_maps_back_to_the_musicbrainz_tag() {
|
||||
assert_eq!(slug_to_tag("hip-hop"), "hip hop");
|
||||
assert_eq!(slug_to_tag("jazz"), "jazz");
|
||||
assert_eq!(slug_to_tag("drum-and-bass"), "drum and bass");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn genre_slug_is_read_from_the_filter() {
|
||||
assert_eq!(
|
||||
genre_slug(&q("", &[("genres", "Hip-Hop")])).as_deref(),
|
||||
Some("hip-hop")
|
||||
);
|
||||
assert_eq!(genre_slug(&q("", &[])), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
//! `musicbrainz-feeder` — the MusicBrainz identity feeder, and the shared MusicBrainz
|
||||
//! client its sibling feeders depend on.
|
||||
//!
|
||||
//! This crate is **lib + bin**. The binary serves the `musicbrainz` plugin
|
||||
//! (`fileType:card`, `contentKind:album|artist`); the library exports
|
||||
//! [`client`] and [`consts`] so `meta-feeder-youtube` can anchor a track on a
|
||||
//! MusicBrainz recording without vendoring a second client.
|
||||
//!
|
||||
//! Everything upstream-agnostic — the redb cache, the rate budget, the
|
||||
//! `0x1006` url locator, `percent_encode`, `normalise_date`, `licence_from_url`
|
||||
//! — lives in `meta-feeder-sdk`, not here.
|
||||
|
||||
pub mod card;
|
||||
pub mod client;
|
||||
pub mod consts;
|
||||
pub mod discovery;
|
||||
pub mod listenbrainz;
|
||||
pub mod musicbrainz;
|
||||
pub mod resolve;
|
||||
@@ -0,0 +1,366 @@
|
||||
//! ListenBrainz client — the **browse** signal.
|
||||
//!
|
||||
//! MusicBrainz knows what exists; it has no opinion about what anyone is
|
||||
//! listening to. ListenBrainz supplies that, from real listen counts, with no
|
||||
//! credential needed for the sitewide and fresh-release feeds. It is the
|
||||
//! `popular:true` / `trending:true` / new-releases answer, exactly as TMDB's
|
||||
//! `/discover` and `/trending` are for the card feeder.
|
||||
//!
|
||||
//! ## Why the parsing here is deliberately lenient
|
||||
//!
|
||||
//! Every other upstream in this crate is parsed into typed structs. This one
|
||||
//! is walked as `serde_json::Value`, on purpose:
|
||||
//!
|
||||
//! - The statistics API had a documented breaking change (`stats_range` →
|
||||
//! `range`) and the payload key differs per endpoint (`artists`,
|
||||
//! `release_groups`, `releases`).
|
||||
//! - The failure mode of a rigid struct here is a **whole empty browse row
|
||||
//! with no error** — the consumer shows a blank wall and there is nothing in
|
||||
//! any log to explain it. That exact class of silent-empty-row failure has
|
||||
//! cost real debugging time elsewhere in this stack.
|
||||
//!
|
||||
//! So each field is read by name with fallbacks, an entry that yields no MBID
|
||||
//! is skipped individually rather than failing the batch, and a response whose
|
||||
//! shape we do not recognise logs loudly at `warn` with the keys it *did*
|
||||
//! carry.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde_json::Value;
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::consts::{LB_API_BASE, LB_STATS_RANGE, LB_TIMEOUT_SECS, USER_AGENT};
|
||||
use meta_feeder_sdk::budget::{Lease, RateBudget};
|
||||
|
||||
/// A work ListenBrainz reports as popular or newly released.
|
||||
///
|
||||
/// Everything needed to build an album card *without a MusicBrainz round
|
||||
/// trip* — which is the whole point: at 1 req/s, resolving twenty
|
||||
/// release-groups by MBID would take twenty seconds.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct PopularWork {
|
||||
pub release_group_mbid: String,
|
||||
pub title: String,
|
||||
pub artist_name: String,
|
||||
pub artist_mbid: Option<String>,
|
||||
/// `(release_mbid, caa_id)` when ListenBrainz already resolved a Cover Art
|
||||
/// Archive image. Saves the consumer a redirect and, more importantly,
|
||||
/// names a cover that is **known to exist** — the release-group front
|
||||
/// endpoint 404s silently for anything unillustrated.
|
||||
pub caa: Option<(String, i64)>,
|
||||
pub release_date: Option<String>,
|
||||
}
|
||||
|
||||
/// An artist ListenBrainz reports as popular.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct PopularArtist {
|
||||
pub artist_mbid: String,
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
pub struct ListenBrainzClient {
|
||||
http: reqwest::Client,
|
||||
base: String,
|
||||
budget: Arc<RateBudget>,
|
||||
}
|
||||
|
||||
impl ListenBrainzClient {
|
||||
pub fn new(budget: Arc<RateBudget>) -> Self {
|
||||
Self::with_base(LB_API_BASE.to_string(), budget)
|
||||
}
|
||||
|
||||
/// Test constructor: point the client at a wiremock server.
|
||||
pub fn with_base(base: String, budget: Arc<RateBudget>) -> Self {
|
||||
Self {
|
||||
http: reqwest::Client::builder()
|
||||
.user_agent(USER_AGENT)
|
||||
.timeout(Duration::from_secs(LB_TIMEOUT_SECS))
|
||||
.build()
|
||||
.expect("build listenbrainz http client"),
|
||||
base: base.trim_end_matches('/').to_string(),
|
||||
budget,
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_json(&self, path_and_query: &str, deadline: Duration) -> Option<Value> {
|
||||
if self.budget.acquire(deadline).await == Lease::DeadlineExceeded {
|
||||
debug!(target: "meta-music", path = %path_and_query, "listenbrainz budget deadline; degrading");
|
||||
return None;
|
||||
}
|
||||
let url = format!("{}/{}", self.base, path_and_query.trim_start_matches('/'));
|
||||
let resp = match self.http.get(&url).send().await {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
warn!(target: "meta-music", %url, error = %e, "listenbrainz request failed");
|
||||
return None;
|
||||
}
|
||||
};
|
||||
if resp.status() == reqwest::StatusCode::TOO_MANY_REQUESTS {
|
||||
let retry = resp
|
||||
.headers()
|
||||
.get(reqwest::header::RETRY_AFTER)
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(2);
|
||||
warn!(target: "meta-music", %url, retry_after_s = retry, "listenbrainz throttled; pausing budget");
|
||||
self.budget.note_throttled(Duration::from_secs(retry));
|
||||
return None;
|
||||
}
|
||||
if !resp.status().is_success() {
|
||||
warn!(target: "meta-music", %url, status = %resp.status(), "listenbrainz non-2xx");
|
||||
return None;
|
||||
}
|
||||
resp.json::<Value>().await.ok()
|
||||
}
|
||||
|
||||
/// Sitewide top release-groups over [`LB_STATS_RANGE`].
|
||||
pub async fn popular_release_groups(&self, count: usize, deadline: Duration) -> Vec<PopularWork> {
|
||||
let q = format!(
|
||||
"stats/sitewide/release-groups?range={LB_STATS_RANGE}&count={}",
|
||||
count.clamp(1, 100)
|
||||
);
|
||||
let Some(v) = self.get_json(&q, deadline).await else {
|
||||
return Vec::new();
|
||||
};
|
||||
parse_works(&v, "release_groups")
|
||||
}
|
||||
|
||||
/// Newly released works. `days` bounds how far back "new" reaches.
|
||||
pub async fn fresh_releases(&self, days: u32, deadline: Duration) -> Vec<PopularWork> {
|
||||
let q = format!(
|
||||
"explore/fresh-releases?days={}&sort=release_date&past=true&future=false",
|
||||
days.clamp(1, 90)
|
||||
);
|
||||
let Some(v) = self.get_json(&q, deadline).await else {
|
||||
return Vec::new();
|
||||
};
|
||||
parse_works(&v, "releases")
|
||||
}
|
||||
|
||||
/// Sitewide top artists over [`LB_STATS_RANGE`].
|
||||
pub async fn popular_artists(&self, count: usize, deadline: Duration) -> Vec<PopularArtist> {
|
||||
let q = format!(
|
||||
"stats/sitewide/artists?range={LB_STATS_RANGE}&count={}",
|
||||
count.clamp(1, 100)
|
||||
);
|
||||
let Some(v) = self.get_json(&q, deadline).await else {
|
||||
return Vec::new();
|
||||
};
|
||||
parse_artists(&v)
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the first present string field from a set of candidate names.
|
||||
fn pick_str<'a>(obj: &'a Value, names: &[&str]) -> Option<&'a str> {
|
||||
names
|
||||
.iter()
|
||||
.find_map(|n| obj.get(*n).and_then(Value::as_str))
|
||||
.map(str::trim)
|
||||
.filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
/// The payload array under `payload.<key>`, tolerating a bare top-level array.
|
||||
fn payload_array<'a>(v: &'a Value, key: &str) -> Option<&'a Vec<Value>> {
|
||||
v.get("payload")
|
||||
.and_then(|p| p.get(key))
|
||||
.and_then(Value::as_array)
|
||||
.or_else(|| v.get(key).and_then(Value::as_array))
|
||||
.or_else(|| v.as_array())
|
||||
}
|
||||
|
||||
/// Parse a works payload (`release_groups` for stats, `releases` for fresh).
|
||||
pub fn parse_works(v: &Value, key: &str) -> Vec<PopularWork> {
|
||||
let Some(items) = payload_array(v, key) else {
|
||||
warn!(
|
||||
target: "meta-music",
|
||||
expected = key,
|
||||
got = ?v.get("payload").map(|p| p.as_object().map(|o| o.keys().cloned().collect::<Vec<_>>())),
|
||||
"listenbrainz payload has no recognised works array; browse row will be empty"
|
||||
);
|
||||
return Vec::new();
|
||||
};
|
||||
let mut out = Vec::with_capacity(items.len());
|
||||
for it in items {
|
||||
// The release-group MBID is the identity an album card is addressed
|
||||
// by. An entry without one cannot become a card at all, so skip it
|
||||
// individually rather than failing the batch — a partial row beats no
|
||||
// row.
|
||||
let Some(rg) = pick_str(it, &["release_group_mbid", "release_group_id"]) else {
|
||||
continue;
|
||||
};
|
||||
let Some(title) = pick_str(
|
||||
it,
|
||||
&["release_group_name", "release_name", "title", "name"],
|
||||
) else {
|
||||
continue;
|
||||
};
|
||||
let artist_name = pick_str(
|
||||
it,
|
||||
&["artist_credit_name", "artist_name", "artist"],
|
||||
)
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
// `artist_mbids` is a list on the stats endpoints and absent on some
|
||||
// fresh-release entries; take the first.
|
||||
let artist_mbid = pick_str(it, &["artist_mbid"])
|
||||
.map(str::to_string)
|
||||
.or_else(|| {
|
||||
it.get("artist_mbids")
|
||||
.and_then(Value::as_array)
|
||||
.and_then(|a| a.first())
|
||||
.and_then(Value::as_str)
|
||||
.map(str::to_string)
|
||||
})
|
||||
.filter(|s| !s.is_empty());
|
||||
let caa = match (
|
||||
pick_str(it, &["caa_release_mbid"]),
|
||||
it.get("caa_id").and_then(Value::as_i64),
|
||||
) {
|
||||
(Some(rel), Some(id)) => Some((rel.to_string(), id)),
|
||||
_ => None,
|
||||
};
|
||||
out.push(PopularWork {
|
||||
release_group_mbid: rg.to_string(),
|
||||
title: title.to_string(),
|
||||
artist_name,
|
||||
artist_mbid,
|
||||
caa,
|
||||
release_date: pick_str(it, &["release_date", "date", "first_release_date"])
|
||||
.map(str::to_string),
|
||||
});
|
||||
}
|
||||
// The same work can appear twice across a fresh-release window (one entry
|
||||
// per pressing). A card is per *work*, so collapse on the release-group.
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
out.retain(|w| seen.insert(w.release_group_mbid.clone()));
|
||||
out
|
||||
}
|
||||
|
||||
/// Parse a sitewide-artists payload.
|
||||
pub fn parse_artists(v: &Value) -> Vec<PopularArtist> {
|
||||
let Some(items) = payload_array(v, "artists") else {
|
||||
warn!(
|
||||
target: "meta-music",
|
||||
"listenbrainz payload has no recognised artists array; browse row will be empty"
|
||||
);
|
||||
return Vec::new();
|
||||
};
|
||||
let mut out = Vec::with_capacity(items.len());
|
||||
for it in items {
|
||||
let Some(mbid) = pick_str(it, &["artist_mbid"]).or_else(|| {
|
||||
it.get("artist_mbids")
|
||||
.and_then(Value::as_array)
|
||||
.and_then(|a| a.first())
|
||||
.and_then(Value::as_str)
|
||||
}) else {
|
||||
continue;
|
||||
};
|
||||
let Some(name) = pick_str(it, &["artist_name", "name"]) else {
|
||||
continue;
|
||||
};
|
||||
out.push(PopularArtist {
|
||||
artist_mbid: mbid.to_string(),
|
||||
name: name.to_string(),
|
||||
});
|
||||
}
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
out.retain(|a| seen.insert(a.artist_mbid.clone()));
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn parses_the_sitewide_release_groups_shape() {
|
||||
let v = json!({"payload": {"release_groups": [{
|
||||
"release_group_mbid": "rg-1",
|
||||
"release_group_name": "Kind of Blue",
|
||||
"artist_name": "Miles Davis",
|
||||
"artist_mbids": ["artist-1"],
|
||||
"caa_id": 12345,
|
||||
"caa_release_mbid": "rel-1",
|
||||
"listen_count": 900
|
||||
}]}});
|
||||
let got = parse_works(&v, "release_groups");
|
||||
assert_eq!(got.len(), 1);
|
||||
assert_eq!(got[0].release_group_mbid, "rg-1");
|
||||
assert_eq!(got[0].title, "Kind of Blue");
|
||||
assert_eq!(got[0].artist_mbid.as_deref(), Some("artist-1"));
|
||||
assert_eq!(got[0].caa, Some(("rel-1".to_string(), 12345)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_the_fresh_releases_shape() {
|
||||
let v = json!({"payload": {"releases": [{
|
||||
"release_group_mbid": "rg-2",
|
||||
"release_name": "New Thing",
|
||||
"artist_credit_name": "Somebody",
|
||||
"release_date": "2026-08-01"
|
||||
}]}});
|
||||
let got = parse_works(&v, "releases");
|
||||
assert_eq!(got.len(), 1);
|
||||
assert_eq!(got[0].artist_name, "Somebody");
|
||||
assert_eq!(got[0].release_date.as_deref(), Some("2026-08-01"));
|
||||
assert!(got[0].caa.is_none(), "no caa fields ⇒ no resolved cover");
|
||||
}
|
||||
|
||||
/// One malformed entry must not empty the row — a partial browse row is
|
||||
/// strictly better than a blank wall with no error.
|
||||
#[test]
|
||||
fn a_bad_entry_is_skipped_not_fatal() {
|
||||
let v = json!({"payload": {"release_groups": [
|
||||
{"nothing": "useful"},
|
||||
{"release_group_mbid": "rg-ok", "release_group_name": "Fine", "artist_name": "A"}
|
||||
]}});
|
||||
let got = parse_works(&v, "release_groups");
|
||||
assert_eq!(got.len(), 1);
|
||||
assert_eq!(got[0].release_group_mbid, "rg-ok");
|
||||
}
|
||||
|
||||
/// A work with several pressings in the window is one card, not three.
|
||||
#[test]
|
||||
fn works_are_deduplicated_by_release_group() {
|
||||
let v = json!({"payload": {"releases": [
|
||||
{"release_group_mbid": "rg-3", "release_name": "Album", "artist_credit_name": "A"},
|
||||
{"release_group_mbid": "rg-3", "release_name": "Album (Deluxe)", "artist_credit_name": "A"}
|
||||
]}});
|
||||
assert_eq!(parse_works(&v, "releases").len(), 1);
|
||||
}
|
||||
|
||||
/// Shape tolerance: a bare top-level array, or an unwrapped key, still
|
||||
/// parses. This is the guard against the next breaking change.
|
||||
#[test]
|
||||
fn tolerates_an_unwrapped_payload() {
|
||||
let unwrapped = json!({"release_groups": [
|
||||
{"release_group_mbid": "rg-4", "release_group_name": "T", "artist_name": "A"}
|
||||
]});
|
||||
assert_eq!(parse_works(&unwrapped, "release_groups").len(), 1);
|
||||
|
||||
let bare = json!([{"release_group_mbid": "rg-5", "release_group_name": "T", "artist_name": "A"}]);
|
||||
assert_eq!(parse_works(&bare, "release_groups").len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_artists_and_skips_mbid_less_entries() {
|
||||
let v = json!({"payload": {"artists": [
|
||||
{"artist_name": "No MBID"},
|
||||
{"artist_name": "Real", "artist_mbid": "a-1"}
|
||||
]}});
|
||||
let got = parse_artists(&v);
|
||||
assert_eq!(got.len(), 1);
|
||||
assert_eq!(got[0].name, "Real");
|
||||
}
|
||||
|
||||
/// An unrecognised payload yields an empty row rather than a panic — and
|
||||
/// the caller logs it.
|
||||
#[test]
|
||||
fn an_unrecognised_payload_is_empty_not_fatal() {
|
||||
let v = json!({"payload": {"something_else": 1}});
|
||||
assert!(parse_works(&v, "release_groups").is_empty());
|
||||
assert!(parse_artists(&v).is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
//! `musicbrainz-feeder` — the MusicBrainz identity feeder.
|
||||
//!
|
||||
//! Serves `fileType:card` for `contentKind:album|artist`. No credential:
|
||||
//! MusicBrainz, the Cover Art Archive and ListenBrainz all serve this traffic
|
||||
//! unauthenticated, so a freshly deployed feeder answers on its first boot.
|
||||
//!
|
||||
//! Env:
|
||||
//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`)
|
||||
//! - `META_FEEDER_STATE_DIR` — per-plugin cache root (default `/data/meta-feeder`)
|
||||
//! - `CARD_TOP_N`, `CARD_DISCOVERY_N` — optional result-sizing seeds
|
||||
//! - `RUST_LOG` — tracing filter (default `info`)
|
||||
|
||||
use std::net::SocketAddr;
|
||||
|
||||
use meta_feeder_sdk::plugin::FeederPlugin;
|
||||
use meta_feeder_sdk::serve_feeders;
|
||||
use musicbrainz_feeder::musicbrainz::MusicBrainzCardPlugin;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
|
||||
.init();
|
||||
|
||||
let listen: SocketAddr = std::env::var("META_FEEDER_HTTP_LISTEN")
|
||||
.unwrap_or_else(|_| "0.0.0.0:8080".to_string())
|
||||
.parse()?;
|
||||
let state_dir =
|
||||
std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string());
|
||||
|
||||
let plugins: Vec<Box<dyn FeederPlugin>> = vec![Box::new(MusicBrainzCardPlugin::new())];
|
||||
serve_feeders(plugins, state_dir, listen).await
|
||||
}
|
||||
@@ -0,0 +1,477 @@
|
||||
//! The `musicbrainz` upstream — a **card** feeder plugin.
|
||||
//!
|
||||
//! Answers the discovery half of gateway search for the music domain: free
|
||||
//! text or a browse marker in, `fileType=card` records out. It never calls an
|
||||
//! indexer and never serves bytes — a card query costs at most one cached
|
||||
//! MusicBrainz search, so the expensive retrieval budget is spent later,
|
||||
//! entirely on the single card the user clicks.
|
||||
//!
|
||||
//! The Cover Art Archive and ListenBrainz are **clients of this plugin**, not
|
||||
//! separate plugins: neither is independently queryable (a cover is addressed
|
||||
//! by a MusicBrainz id, and a popularity chart is a ranking *of* MusicBrainz
|
||||
//! works), so registering them as routable upstreams would advertise
|
||||
//! capabilities nothing can ask for.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use meta_feeder_sdk::config::ConfigSchema;
|
||||
use meta_feeder_sdk::plugin::{ConfigError, FeederPlugin, GatewayQuery, HashKind, HashOutcome};
|
||||
use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, Hash, PluginHealth};
|
||||
use tracing::warn;
|
||||
|
||||
use meta_feeder_sdk::cache::MidhashCache;
|
||||
use crate::card::{split_record_id, Card, CardKind, CardSource};
|
||||
use crate::consts::{
|
||||
DEFAULT_CARD_DISCOVERY_N, DEFAULT_CARD_TOP_N, LB_BURST, LB_RATE_PER_SEC,
|
||||
MB_BURST, MB_DISCOVERY_WAIT_DEADLINE_SECS, MB_RATE_PER_SEC, MB_WAIT_DEADLINE_SECS,
|
||||
};
|
||||
use crate::listenbrainz::ListenBrainzClient;
|
||||
use crate::client::MusicBrainzClient;
|
||||
use meta_feeder_sdk::budget::RateBudget;
|
||||
use crate::resolve::Resolver;
|
||||
|
||||
/// Operator config, supplied via the feeder's own web form.
|
||||
///
|
||||
/// **No API key appears here, and that is the point.** MusicBrainz, the Cover
|
||||
/// Art Archive and ListenBrainz all serve this feeder's traffic without a
|
||||
/// credential, so unlike the TMDB card feeder there is no soft-skip path and
|
||||
/// no "configure me first" state: a freshly deployed music feeder answers
|
||||
/// queries on its first boot. That is what makes a cold meta-listen client
|
||||
/// show a populated board with nothing configured.
|
||||
#[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
|
||||
pub struct MusicBrainzConfig {
|
||||
#[serde(default)]
|
||||
pub card_top_n: Option<usize>,
|
||||
#[serde(default)]
|
||||
pub card_discovery_n: Option<usize>,
|
||||
}
|
||||
|
||||
impl MusicBrainzConfig {
|
||||
/// Env seed, used on first boot before any `config.json` exists.
|
||||
pub fn from_env() -> Self {
|
||||
Self {
|
||||
card_top_n: std::env::var("CARD_TOP_N").ok().and_then(|v| v.parse().ok()),
|
||||
card_discovery_n: std::env::var("CARD_DISCOVERY_N")
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct MusicBrainzCardPlugin {
|
||||
config: MusicBrainzConfig,
|
||||
resolver: Option<Resolver>,
|
||||
top_n: usize,
|
||||
discovery_n: usize,
|
||||
/// Test hook: `(musicbrainz_base, listenbrainz_base)` overriding the real
|
||||
/// endpoints so the contract test can point at a wiremock server.
|
||||
api_bases: Option<(String, String)>,
|
||||
}
|
||||
|
||||
impl Default for MusicBrainzCardPlugin {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl MusicBrainzCardPlugin {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
config: MusicBrainzConfig::from_env(),
|
||||
resolver: None,
|
||||
top_n: DEFAULT_CARD_TOP_N,
|
||||
discovery_n: DEFAULT_CARD_DISCOVERY_N,
|
||||
api_bases: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Test constructor: endpoint overrides, so the contract test can drive the
|
||||
/// real router against a wiremock upstream.
|
||||
pub fn with_api_bases(mb_base: String, lb_base: String) -> Self {
|
||||
let mut p = Self::new();
|
||||
p.api_bases = Some((mb_base, lb_base));
|
||||
p
|
||||
}
|
||||
|
||||
/// File wins over env, exactly like every other feeder. No hot reload —
|
||||
/// bounce the feeder.
|
||||
fn load_config(&mut self, cache_dir: &Path) {
|
||||
let path = cache_dir.join("config.json");
|
||||
if let Ok(bytes) = std::fs::read(&path) {
|
||||
if let Ok(file_cfg) = serde_json::from_slice::<MusicBrainzConfig>(&bytes) {
|
||||
if file_cfg.card_top_n.is_some() {
|
||||
self.config.card_top_n = file_cfg.card_top_n;
|
||||
}
|
||||
if file_cfg.card_discovery_n.is_some() {
|
||||
self.config.card_discovery_n = file_cfg.card_discovery_n;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resolver(&self) -> Option<&Resolver> {
|
||||
self.resolver.as_ref()
|
||||
}
|
||||
}
|
||||
|
||||
/// The deadline a query somebody is waiting on may spend on the budget.
|
||||
fn search_deadline() -> Duration {
|
||||
Duration::from_secs(MB_WAIT_DEADLINE_SECS)
|
||||
}
|
||||
|
||||
/// The (much shorter) deadline a browse row may spend. See
|
||||
/// `consts::MB_DISCOVERY_WAIT_DEADLINE_SECS`.
|
||||
fn browse_deadline() -> Duration {
|
||||
Duration::from_secs(MB_DISCOVERY_WAIT_DEADLINE_SECS)
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl FeederPlugin for MusicBrainzCardPlugin {
|
||||
fn upstream_id(&self) -> &'static str {
|
||||
"musicbrainz"
|
||||
}
|
||||
|
||||
/// **The routing declaration.** `card` on the fileType axis is what makes
|
||||
/// meta-search fan a `fileType:card` query out to this gateway;
|
||||
/// `album`/`artist` on the contentKind axis is what keeps a *video* card
|
||||
/// query (`contentKind:series`) from reaching it. Both are matched
|
||||
/// dynamically against the heartbeat by
|
||||
/// `meta-search`'s `upstreams_for_query_from_raw`, so adding this tier
|
||||
/// needs no meta-search change.
|
||||
fn served_file_types(&self) -> &'static [&'static str] {
|
||||
&["card"]
|
||||
}
|
||||
|
||||
fn served_content_kinds(&self) -> &'static [&'static str] {
|
||||
&["album", "artist"]
|
||||
}
|
||||
|
||||
fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> {
|
||||
self.load_config(cache_dir);
|
||||
self.top_n = self
|
||||
.config
|
||||
.card_top_n
|
||||
.filter(|n| *n > 0)
|
||||
.unwrap_or(DEFAULT_CARD_TOP_N);
|
||||
self.discovery_n = self
|
||||
.config
|
||||
.card_discovery_n
|
||||
.filter(|n| *n > 0)
|
||||
.unwrap_or(DEFAULT_CARD_DISCOVERY_N);
|
||||
|
||||
let cache = MidhashCache::open(cache_dir).map_err(|e| ConfigError::Other {
|
||||
plugin: "musicbrainz",
|
||||
source: anyhow::anyhow!("open music cache: {e}"),
|
||||
})?;
|
||||
|
||||
// Two independent budgets. A shared one would either throttle
|
||||
// ListenBrainz to MusicBrainz's 1/s, or let MusicBrainz burst past a
|
||||
// ceiling whose breach blocks the whole source IP.
|
||||
let mb_budget = RateBudget::new(MB_RATE_PER_SEC, MB_BURST);
|
||||
let lb_budget = RateBudget::new(LB_RATE_PER_SEC, LB_BURST);
|
||||
|
||||
let (mb, lb) = match &self.api_bases {
|
||||
Some((mb_base, lb_base)) => (
|
||||
MusicBrainzClient::with_base(mb_base.clone(), mb_budget, cache),
|
||||
ListenBrainzClient::with_base(lb_base.clone(), lb_budget),
|
||||
),
|
||||
None => (
|
||||
MusicBrainzClient::new(mb_budget, cache),
|
||||
ListenBrainzClient::new(lb_budget),
|
||||
),
|
||||
};
|
||||
self.resolver = Some(Resolver::new(Arc::new(mb), Arc::new(lb)));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn health(&self) -> PluginHealth {
|
||||
match self.resolver {
|
||||
Some(_) => PluginHealth::Ok,
|
||||
// Only reachable before `configure` ran, or if the cache failed to
|
||||
// open. Unlike the TMDB card feeder there is no credential to be
|
||||
// missing, so this is a real fault rather than an opt-in soft-skip.
|
||||
None => PluginHealth::Degraded {
|
||||
reason: "configure() not yet called (or the local cache failed to open)".to_string(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_query(
|
||||
&self,
|
||||
query: &GatewayQuery,
|
||||
max_results: usize,
|
||||
) -> Result<Vec<DiscoveryRecord>, GatewayError> {
|
||||
// Layer A early-return: this plugin only ever serves music cards.
|
||||
if !meta_feeder_sdk::query_eval::query_accepts_plugin(
|
||||
query,
|
||||
self.served_file_types(),
|
||||
self.served_content_kinds(),
|
||||
) {
|
||||
// One misconfiguration lands here often enough to deserve a name: a
|
||||
// browse row copied from the *video* card tier says
|
||||
// `contentKind:series`, which is not in `served_content_kinds`, so
|
||||
// the row is rejected here before any upstream is touched and the
|
||||
// operator sees an empty wall with no error anywhere.
|
||||
if crate::discovery::is_browse_query(query) {
|
||||
warn!(
|
||||
target: "meta-music",
|
||||
content_kind = ?query.filters.get("contentKind"),
|
||||
"browse row rejected by the served-kinds gate: this feeder \
|
||||
serves contentKind:album / contentKind:artist"
|
||||
);
|
||||
}
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let Some(resolver) = self.resolver() else {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
|
||||
// Three shapes, in precedence order:
|
||||
// 1. an explicit MBID filter — the direct "give me this work" lookup
|
||||
// a deep link (or the album/artist page) uses;
|
||||
// 2. a keyword-less browse query (`popular:true contentKind:album`);
|
||||
// 3. free text — the principal search.
|
||||
let cards: Vec<Card> = if let Some(mbid) = first_filter(query, "mbReleaseGroupId") {
|
||||
resolver
|
||||
.album_card_by_mbid(&mbid, search_deadline())
|
||||
.await
|
||||
.into_iter()
|
||||
.collect()
|
||||
} else if let Some(mbid) = first_filter(query, "mbArtistId") {
|
||||
// An artist id means one of two questions: "who is this artist?"
|
||||
// (an artist card) or "what did they release?" (their albums). The
|
||||
// requested `contentKind` decides, and album is the default — a
|
||||
// detail page asks for the discography far more often than it asks
|
||||
// for the artist tile it already has.
|
||||
if wants_kind(query, CardKind::Artist) && !wants_kind(query, CardKind::Album) {
|
||||
resolver
|
||||
.artist_card_by_mbid(&mbid, search_deadline())
|
||||
.await
|
||||
.into_iter()
|
||||
.collect()
|
||||
} else {
|
||||
resolver
|
||||
.album_cards_for_artist(&mbid, max_results.max(1), search_deadline())
|
||||
.await
|
||||
}
|
||||
} else if crate::discovery::is_browse_query(query) {
|
||||
let n = self.discovery_n.min(max_results.max(1));
|
||||
crate::discovery::cards_for_browse(resolver, query, n, browse_deadline()).await
|
||||
} else {
|
||||
let free_text = query.free_text.trim();
|
||||
if free_text.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
let n = self.top_n.min(max_results.max(1));
|
||||
let mut cards = Vec::new();
|
||||
if wants_kind(query, CardKind::Album) {
|
||||
cards.extend(resolver.album_cards_for_text(free_text, n, search_deadline()).await);
|
||||
}
|
||||
if wants_kind(query, CardKind::Artist) {
|
||||
cards.extend(resolver.artist_cards_for_text(free_text, n, search_deadline()).await);
|
||||
}
|
||||
cards
|
||||
};
|
||||
|
||||
Ok(cards
|
||||
.iter()
|
||||
.filter_map(|c| c.to_record(&query.filters))
|
||||
.take(max_results)
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Resolve a card `record_id` back into its locator CID.
|
||||
///
|
||||
/// Not a stub for a byte-less plugin — this *is* the card path. The outcome
|
||||
/// carries `bytes: None`, so the gateway core's three-branch auto-store
|
||||
/// routes it to metadata-only: the record lands in meta-core, nothing is
|
||||
/// written to WebDAV, and nothing is seeded to bitswap (seeding is gated on
|
||||
/// `Sha2_256`). The record IS the payload.
|
||||
///
|
||||
/// The CID needs no network call — it is a pure function of the record id.
|
||||
/// The card is re-resolved anyway so the stored record is complete, but a
|
||||
/// resolution failure still yields the CID, because the identity is
|
||||
/// derivable regardless.
|
||||
async fn compute_outcomes(&self, record_id: &str) -> Result<Vec<HashOutcome>, GatewayError> {
|
||||
let (source, mbid) = split_record_id(record_id).ok_or_else(|| {
|
||||
GatewayError::Permanent(format!("malformed card record_id '{record_id}'"))
|
||||
})?;
|
||||
if source != CardSource::MusicBrainz.as_str() {
|
||||
return Err(GatewayError::NotFound);
|
||||
}
|
||||
let hash = meta_feeder_sdk::hash::compute_card_cid(source, mbid).ok_or_else(|| {
|
||||
GatewayError::Permanent(format!("card id too long to encode: '{record_id}'"))
|
||||
})?;
|
||||
|
||||
// ⚠ The record id carries no kind (see `card.rs` — the id IS the CID
|
||||
// preimage, so a `album:`/`artist:` prefix would re-mint every
|
||||
// address), so which entity this MBID names has to be discovered.
|
||||
// Release-group first: albums outnumber artists in every board and
|
||||
// every search, so the common case costs one call rather than two.
|
||||
let record = match self.resolver() {
|
||||
Some(r) => match r.album_card_by_mbid(mbid, search_deadline()).await {
|
||||
Some(c) => c.to_record(&Default::default()),
|
||||
None => r
|
||||
.artist_card_by_mbid(mbid, search_deadline())
|
||||
.await
|
||||
.and_then(|c| c.to_record(&Default::default())),
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
|
||||
Ok(vec![HashOutcome {
|
||||
hash: Hash(hash),
|
||||
hash_kind: HashKind::CardLocator,
|
||||
bytes: None,
|
||||
record,
|
||||
file_extension: None,
|
||||
}])
|
||||
}
|
||||
|
||||
fn config_schema(&self) -> ConfigSchema {
|
||||
use meta_feeder_sdk::config::{ConfigField as F, ConfigSchema};
|
||||
ConfigSchema {
|
||||
fields: vec![
|
||||
F::number("card_top_n", "Cards per search").with_help(
|
||||
"How many works a free-text search returns. Blank keeps the \
|
||||
built-in default (10). Costs only MusicBrainz calls — no \
|
||||
indexer budget is spent until a card is clicked.",
|
||||
),
|
||||
F::number("card_discovery_n", "Cards per browse row").with_help(
|
||||
"How many cards a keyword-less browse row (popular / trending / \
|
||||
fresh) returns. Blank keeps the built-in default (20). A row \
|
||||
costs one ListenBrainz call; the feeder over-fetches because \
|
||||
it cannot tell in advance which works have cover art.",
|
||||
),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
fn config_values(&self) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"card_top_n": self.config.card_top_n,
|
||||
"card_discovery_n": self.config.card_discovery_n,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The first value of a structured filter, trimmed and non-empty.
|
||||
fn first_filter(query: &GatewayQuery, key: &str) -> Option<String> {
|
||||
query
|
||||
.filters
|
||||
.get(key)
|
||||
.and_then(|v| v.first())
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
}
|
||||
|
||||
/// Does the query ask for this card kind? A query with no `contentKind` filter
|
||||
/// wants everything this plugin serves.
|
||||
fn wants_kind(query: &GatewayQuery, kind: CardKind) -> bool {
|
||||
match query.filters.get("contentKind") {
|
||||
None => true,
|
||||
Some(values) if values.is_empty() => true,
|
||||
Some(values) => values
|
||||
.iter()
|
||||
.any(|v| v.trim().eq_ignore_ascii_case(kind.content_kind())),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
const RG: &str = "f5093c06-23e3-404f-aeaa-40f72885ee3a";
|
||||
|
||||
/// The routing declaration is load-bearing: `card` here is the only reason
|
||||
/// meta-search fans a `fileType:card` query out to this gateway, and the
|
||||
/// content kinds are what keep a video card row from landing here.
|
||||
#[test]
|
||||
fn declares_the_card_file_type_and_the_music_work_kinds() {
|
||||
let p = MusicBrainzCardPlugin::new();
|
||||
assert_eq!(p.upstream_id(), "musicbrainz");
|
||||
assert_eq!(p.served_file_types(), &["card"]);
|
||||
assert_eq!(p.served_content_kinds(), &["album", "artist"]);
|
||||
}
|
||||
|
||||
/// A card CID is derivable with no network and no resolver, so
|
||||
/// `compute_outcomes` must still produce one on an unconfigured plugin —
|
||||
/// only the record is missing.
|
||||
#[tokio::test]
|
||||
async fn compute_outcomes_yields_a_byteless_locator_without_configuration() {
|
||||
let p = MusicBrainzCardPlugin::new();
|
||||
let outcomes = p
|
||||
.compute_outcomes(&format!("musicbrainz:{RG}"))
|
||||
.await
|
||||
.expect("outcomes");
|
||||
assert_eq!(outcomes.len(), 1);
|
||||
let o = &outcomes[0];
|
||||
assert_eq!(o.hash_kind, HashKind::CardLocator);
|
||||
assert!(o.bytes.is_none(), "a card has no bytes, ever");
|
||||
assert!(o.file_extension.is_none());
|
||||
assert_eq!(
|
||||
o.hash.0,
|
||||
meta_feeder_sdk::hash::compute_card_cid("musicbrainz", RG).unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compute_outcomes_rejects_a_foreign_or_malformed_id() {
|
||||
let p = MusicBrainzCardPlugin::new();
|
||||
assert!(p.compute_outcomes("tmdb:tv:95479").await.is_err());
|
||||
assert!(p.compute_outcomes("nosource").await.is_err());
|
||||
}
|
||||
|
||||
/// Layer-A routing: a query for something this plugin cannot serve returns
|
||||
/// immediately, without touching any upstream.
|
||||
#[tokio::test]
|
||||
async fn rejects_a_query_for_a_type_it_does_not_serve() {
|
||||
let p = MusicBrainzCardPlugin::new();
|
||||
let mut q = GatewayQuery::from_free_text("miles davis");
|
||||
q.filters
|
||||
.insert("fileType".to_string(), vec!["video".to_string()]);
|
||||
assert!(p.handle_query(&q, 10).await.unwrap().is_empty());
|
||||
}
|
||||
|
||||
/// An unconfigured plugin answers nothing rather than panicking — but it
|
||||
/// reports itself degraded, because unlike the TMDB feeder there is no
|
||||
/// credential that could legitimately be absent.
|
||||
#[tokio::test]
|
||||
async fn an_unconfigured_plugin_is_degraded_and_answers_nothing() {
|
||||
let p = MusicBrainzCardPlugin::new();
|
||||
let q = GatewayQuery::from_free_text("miles davis");
|
||||
assert!(p.handle_query(&q, 10).await.unwrap().is_empty());
|
||||
assert!(matches!(p.health(), PluginHealth::Degraded { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_query_with_no_content_kind_wants_every_served_kind() {
|
||||
let q = GatewayQuery::from_free_text("x");
|
||||
assert!(wants_kind(&q, CardKind::Album));
|
||||
assert!(wants_kind(&q, CardKind::Artist));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_content_kind_filter_selects_one_kind() {
|
||||
let mut q = GatewayQuery::from_free_text("x");
|
||||
q.filters
|
||||
.insert("contentKind".to_string(), vec!["artist".to_string()]);
|
||||
assert!(!wants_kind(&q, CardKind::Album));
|
||||
assert!(wants_kind(&q, CardKind::Artist));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn first_filter_trims_and_rejects_empties() {
|
||||
let mut q = GatewayQuery::from_free_text("x");
|
||||
q.filters
|
||||
.insert("mbReleaseGroupId".to_string(), vec![" rg-1 ".to_string()]);
|
||||
q.filters
|
||||
.insert("mbArtistId".to_string(), vec![" ".to_string()]);
|
||||
assert_eq!(first_filter(&q, "mbReleaseGroupId").as_deref(), Some("rg-1"));
|
||||
assert_eq!(first_filter(&q, "mbArtistId"), None);
|
||||
assert_eq!(first_filter(&q, "absent"), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,487 @@
|
||||
//! MusicBrainz responses → [`Card`]s.
|
||||
//!
|
||||
//! The card feeder's `resolve.rs` in the music domain. Everything here is
|
||||
//! budget-gated through the clients; nothing here talks to the network
|
||||
//! directly.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::card::{Card, CardKind, CardSource, Cover};
|
||||
use crate::consts::MB_MAX_PAGES;
|
||||
use crate::listenbrainz::{ListenBrainzClient, PopularArtist, PopularWork};
|
||||
use crate::client::{
|
||||
primary_artist, render_credit, Artist, MusicBrainzClient, ReleaseGroup, Tag,
|
||||
};
|
||||
|
||||
/// Minimum tag vote count for a tag to be treated as a genre.
|
||||
///
|
||||
/// MusicBrainz tags are open user input, so the long tail is full of one-vote
|
||||
/// noise ("stuff i like", a misspelling, a mood). A floor of 1 net vote is
|
||||
/// enough to remove the accidental and the vandalised while keeping genuinely
|
||||
/// niche genres — the same role the card feeder's `DISCOVERY_MIN_VOTES` plays
|
||||
/// for TMDB, at a far smaller scale because a tag vote is much cheaper to cast
|
||||
/// than a TMDB rating.
|
||||
const MIN_TAG_VOTES: i64 = 1;
|
||||
|
||||
/// Ceiling on genres carried by one card. A heavily-tagged release can have
|
||||
/// forty; past a handful they stop being a facet and start being noise, and
|
||||
/// each one is its own hash field on the record.
|
||||
const MAX_GENRES: usize = 6;
|
||||
|
||||
/// Pick the genre names from a MusicBrainz tag/genre list.
|
||||
///
|
||||
/// `genres` (curated, from `inc=genres`) wins outright when present; `tags`
|
||||
/// (open user input) is the fallback a search-result entry carries. They are
|
||||
/// never merged — a curated list that says "Jazz" should not be diluted by a
|
||||
/// tag list that also says "cool".
|
||||
pub fn genre_names(genres: &[Tag], tags: &[Tag]) -> Vec<String> {
|
||||
let source = if genres.iter().any(|t| !t.name.trim().is_empty()) {
|
||||
genres
|
||||
} else {
|
||||
tags
|
||||
};
|
||||
let mut ranked: Vec<&Tag> = source
|
||||
.iter()
|
||||
.filter(|t| !t.name.trim().is_empty() && t.count >= MIN_TAG_VOTES)
|
||||
.collect();
|
||||
ranked.sort_by(|a, b| b.count.cmp(&a.count).then_with(|| a.name.cmp(&b.name)));
|
||||
ranked
|
||||
.into_iter()
|
||||
.take(MAX_GENRES)
|
||||
.map(|t| title_case(t.name.trim()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// MusicBrainz tags are lowercase by convention (`modal jazz`), while
|
||||
/// `METADATA_KEYS.md`'s `genres/{name}` key-set is written with the name
|
||||
/// verbatim as the upstream publishes it — and every other writer publishes
|
||||
/// display-cased names (`Animation`, `Sci-Fi`). Title-casing here keeps one
|
||||
/// display convention across the whole key-set rather than splitting it by
|
||||
/// which feeder happened to produce the record.
|
||||
/// ⚠ Capitalise after **any** non-alphanumeric character, not just a space.
|
||||
/// Splitting on spaces alone turned the tag `r&b` into `R&b`, which is visibly
|
||||
/// wrong in a genre chip and — since the name is the key-set member itself —
|
||||
/// would have made `genres/R&b` and a correctly-cased `genres/R&B` two
|
||||
/// different members of the same set on the same record.
|
||||
fn title_case(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut at_boundary = true;
|
||||
for c in s.chars() {
|
||||
if at_boundary {
|
||||
out.extend(c.to_uppercase());
|
||||
} else {
|
||||
out.push(c);
|
||||
}
|
||||
at_boundary = !c.is_alphanumeric();
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
pub struct Resolver {
|
||||
pub mb: Arc<MusicBrainzClient>,
|
||||
pub lb: Arc<ListenBrainzClient>,
|
||||
}
|
||||
|
||||
impl Resolver {
|
||||
pub fn new(mb: Arc<MusicBrainzClient>, lb: Arc<ListenBrainzClient>) -> Self {
|
||||
Self { mb, lb }
|
||||
}
|
||||
|
||||
/// An album card from a release-group, with no further upstream calls.
|
||||
///
|
||||
/// The cover is the release-group front endpoint, which may 404 — see
|
||||
/// [`Card::is_displayable`] for why that cannot be predicted here.
|
||||
pub fn album_card(&self, rg: &ReleaseGroup) -> Option<Card> {
|
||||
if rg.id.is_empty() || rg.title.trim().is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(Card {
|
||||
source: CardSource::MusicBrainz,
|
||||
kind: CardKind::Album,
|
||||
mbid: rg.id.clone(),
|
||||
title: rg.title.trim().to_string(),
|
||||
album_artist: Some(render_credit(&rg.artist_credit)).filter(|s| !s.is_empty()),
|
||||
artists: rg
|
||||
.artist_credit
|
||||
.iter()
|
||||
.filter_map(|c| c.artist.as_ref())
|
||||
.map(|a| a.name.clone())
|
||||
.filter(|n| !n.trim().is_empty())
|
||||
.collect(),
|
||||
artist_mbid: primary_artist(&rg.artist_credit).map(|a| a.id.clone()),
|
||||
cover: Some(Cover::ReleaseGroupFront {
|
||||
release_group_mbid: rg.id.clone(),
|
||||
}),
|
||||
disambiguation: rg.disambiguation.clone().filter(|s| !s.trim().is_empty()),
|
||||
genres: genre_names(&rg.genres, &rg.tags),
|
||||
release_date: rg.first_release_date.clone(),
|
||||
track_count: None,
|
||||
disc_count: None,
|
||||
label: None,
|
||||
catalog_number: None,
|
||||
primary_type: rg.primary_type.clone(),
|
||||
secondary_types: rg.secondary_types.clone(),
|
||||
tracks: Vec::new(),
|
||||
streaming_links: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// An album card from a ListenBrainz browse entry, with no MusicBrainz
|
||||
/// call at all.
|
||||
///
|
||||
/// This is what makes a browse row affordable: at 1 req/s, resolving
|
||||
/// twenty release-groups by MBID would take twenty seconds and every
|
||||
/// consumer-side idle cutoff would fire first. A ListenBrainz entry
|
||||
/// already carries the title, the artist credit, both MBIDs and often a
|
||||
/// *resolved* cover id — everything a renderable card needs.
|
||||
///
|
||||
/// The price is a thinner card: no genres, no track count. Both are
|
||||
/// resolved on the detail path, which is where anything reads them.
|
||||
pub fn album_card_from_popular(&self, w: &PopularWork) -> Option<Card> {
|
||||
if w.release_group_mbid.is_empty() || w.title.trim().is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(Card {
|
||||
source: CardSource::MusicBrainz,
|
||||
kind: CardKind::Album,
|
||||
mbid: w.release_group_mbid.clone(),
|
||||
title: w.title.trim().to_string(),
|
||||
album_artist: Some(w.artist_name.clone()).filter(|s| !s.trim().is_empty()),
|
||||
artists: if w.artist_name.trim().is_empty() {
|
||||
Vec::new()
|
||||
} else {
|
||||
vec![w.artist_name.clone()]
|
||||
},
|
||||
artist_mbid: w.artist_mbid.clone(),
|
||||
cover: Some(match &w.caa {
|
||||
Some((release_mbid, caa_id)) => Cover::Resolved {
|
||||
release_mbid: release_mbid.clone(),
|
||||
caa_id: *caa_id,
|
||||
},
|
||||
None => Cover::ReleaseGroupFront {
|
||||
release_group_mbid: w.release_group_mbid.clone(),
|
||||
},
|
||||
}),
|
||||
disambiguation: None,
|
||||
genres: Vec::new(),
|
||||
release_date: w.release_date.clone(),
|
||||
track_count: None,
|
||||
disc_count: None,
|
||||
label: None,
|
||||
catalog_number: None,
|
||||
primary_type: None,
|
||||
secondary_types: Vec::new(),
|
||||
tracks: Vec::new(),
|
||||
streaming_links: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// An artist card.
|
||||
///
|
||||
/// ⚠ **The image is borrowed, and that is a real compromise.** Neither
|
||||
/// MusicBrainz nor the Cover Art Archive holds artist photographs — the CAA
|
||||
/// is keyed by release and release-group only, and MusicBrainz does not
|
||||
/// host images at all for licensing reasons. The correct source is Wikidata
|
||||
/// (`P18`) reached through MusicBrainz `inc=url-rels`, which costs **two
|
||||
/// extra requests per artist** against a 1 req/s budget: a twenty-artist
|
||||
/// row would take forty seconds and every idle cutoff would fire first.
|
||||
///
|
||||
/// So a browse-row artist card borrows a cover from one of that artist's
|
||||
/// releases, which is what the data actually supports at this cost. An
|
||||
/// artist for whom no cover is in hand gets no card rather than a blank
|
||||
/// tile. If artist photography becomes load-bearing, the Wikidata path
|
||||
/// belongs on the *detail* route, where one artist justifies two calls.
|
||||
pub fn artist_card(&self, a: &Artist, cover: Option<Cover>) -> Option<Card> {
|
||||
if a.id.is_empty() || a.name.trim().is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(Card {
|
||||
source: CardSource::MusicBrainz,
|
||||
kind: CardKind::Artist,
|
||||
mbid: a.id.clone(),
|
||||
title: a.name.trim().to_string(),
|
||||
album_artist: None,
|
||||
artists: vec![a.name.clone()],
|
||||
artist_mbid: None,
|
||||
cover,
|
||||
disambiguation: a.disambiguation.clone().filter(|s| !s.trim().is_empty()),
|
||||
genres: genre_names(&a.genres, &a.tags),
|
||||
release_date: None,
|
||||
track_count: None,
|
||||
disc_count: None,
|
||||
label: None,
|
||||
catalog_number: None,
|
||||
primary_type: a.artist_type.clone(),
|
||||
secondary_types: Vec::new(),
|
||||
tracks: Vec::new(),
|
||||
streaming_links: Vec::new(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Free-text album search.
|
||||
pub async fn album_cards_for_text(
|
||||
&self,
|
||||
text: &str,
|
||||
n: usize,
|
||||
deadline: Duration,
|
||||
) -> Vec<Card> {
|
||||
self.mb
|
||||
.search_release_groups(text, n, deadline)
|
||||
.await
|
||||
.iter()
|
||||
.filter_map(|rg| self.album_card(rg))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Free-text artist search.
|
||||
///
|
||||
/// Each hit needs a cover borrowed from one of its releases (see
|
||||
/// [`Resolver::artist_card`]), which is one browse call per artist against
|
||||
/// a 1 req/s budget — so this is deliberately capped far below the album
|
||||
/// search's breadth. A search box wants a handful of artists, not twenty.
|
||||
pub async fn artist_cards_for_text(
|
||||
&self,
|
||||
text: &str,
|
||||
n: usize,
|
||||
deadline: Duration,
|
||||
) -> Vec<Card> {
|
||||
const MAX_ARTIST_COVER_LOOKUPS: usize = 3;
|
||||
let hits = self.mb.search_artists(text, n, deadline).await;
|
||||
let mut out = Vec::with_capacity(hits.len());
|
||||
for (i, a) in hits.iter().enumerate() {
|
||||
let cover = if i < MAX_ARTIST_COVER_LOOKUPS {
|
||||
self.cover_for_artist(&a.id, deadline).await
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(card) = self.artist_card(a, cover) {
|
||||
out.push(card);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Artist cards for a ListenBrainz popularity list.
|
||||
///
|
||||
/// Same cover problem, same bounded answer: only the head of the row pays
|
||||
/// for a cover lookup. A short row of real tiles beats a long row of
|
||||
/// tiles the gate will hide anyway.
|
||||
pub async fn artist_cards_from_popular(
|
||||
&self,
|
||||
artists: &[PopularArtist],
|
||||
n: usize,
|
||||
deadline: Duration,
|
||||
) -> Vec<Card> {
|
||||
let mut out = Vec::with_capacity(n);
|
||||
for pa in artists.iter().take(n) {
|
||||
let Some(cover) = self.cover_for_artist(&pa.artist_mbid, deadline).await else {
|
||||
continue;
|
||||
};
|
||||
let a = Artist {
|
||||
id: pa.artist_mbid.clone(),
|
||||
name: pa.name.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(card) = self.artist_card(&a, Some(cover)) {
|
||||
out.push(card);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Borrow a cover from one of an artist's release-groups. One call.
|
||||
async fn cover_for_artist(&self, artist_mbid: &str, deadline: Duration) -> Option<Cover> {
|
||||
let rgs = self.mb.release_groups_by_artist(artist_mbid, 5, deadline).await;
|
||||
// Prefer a full album over a single or a compilation: an album cover is
|
||||
// the image people associate with the artist.
|
||||
let pick = rgs
|
||||
.iter()
|
||||
.find(|rg| rg.primary_type.as_deref() == Some("Album"))
|
||||
.or_else(|| rgs.first())?;
|
||||
Some(Cover::ReleaseGroupFront {
|
||||
release_group_mbid: pick.id.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Album cards for a genre tag — the genre row.
|
||||
pub async fn album_cards_for_tag(
|
||||
&self,
|
||||
tag: &str,
|
||||
n: usize,
|
||||
deadline: Duration,
|
||||
) -> Vec<Card> {
|
||||
self.mb
|
||||
.release_groups_by_tag(tag, n, deadline)
|
||||
.await
|
||||
.iter()
|
||||
.filter_map(|rg| self.album_card(rg))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The direct "give me this work" lookup a deep link or a detail page uses.
|
||||
/// Resolves the full card, including the canonical track list.
|
||||
pub async fn album_card_by_mbid(&self, mbid: &str, deadline: Duration) -> Option<Card> {
|
||||
let rg = self.mb.release_group(mbid, deadline).await?;
|
||||
let card = self.album_card(&rg)?;
|
||||
match self.mb.tracklist(mbid, deadline).await {
|
||||
Some(release) => Some(card.with_release_detail(&release)),
|
||||
None => Some(card),
|
||||
}
|
||||
}
|
||||
|
||||
/// The direct artist lookup — the **detail** path.
|
||||
///
|
||||
/// Also collects the artist's official streaming links, which the browse
|
||||
/// paths deliberately do not: it costs one more MusicBrainz call against a
|
||||
/// 1 req/s budget, and a browse row renders no link affordances. Detail
|
||||
/// pages are opened one at a time, so the call is affordable exactly here.
|
||||
pub async fn artist_card_by_mbid(&self, mbid: &str, deadline: Duration) -> Option<Card> {
|
||||
let a = self.mb.artist(mbid, deadline).await?;
|
||||
let cover = self.cover_for_artist(mbid, deadline).await;
|
||||
let mut card = self.artist_card(&a, cover)?;
|
||||
card.streaming_links = self.streaming_links(mbid, deadline).await;
|
||||
Some(card)
|
||||
}
|
||||
|
||||
/// Official "listen elsewhere" URLs from MusicBrainz's artist relations.
|
||||
///
|
||||
/// Both flavours count: `free streaming` (Deezer, Spotify, Yandex) and
|
||||
/// `streaming` (Tidal, Apple, Qobuz, Amazon). YouTube relations are
|
||||
/// excluded — those are the *delegated playback* tier's channel keys, and a
|
||||
/// channel page is a browse target, not a rendition to open.
|
||||
async fn streaming_links(&self, mbid: &str, deadline: Duration) -> Vec<String> {
|
||||
let Some(rels) = self.mb.artist_url_relations(mbid, deadline).await else {
|
||||
return Vec::new();
|
||||
};
|
||||
let mut out = Vec::new();
|
||||
for (ty, url) in rels {
|
||||
let t = ty.to_ascii_lowercase();
|
||||
if t != "streaming" && t != "free streaming" {
|
||||
continue;
|
||||
}
|
||||
if !(url.starts_with("http://") || url.starts_with("https://")) {
|
||||
continue;
|
||||
}
|
||||
if !out.contains(&url) {
|
||||
out.push(url);
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// An artist's discography as album cards.
|
||||
pub async fn album_cards_for_artist(
|
||||
&self,
|
||||
artist_mbid: &str,
|
||||
n: usize,
|
||||
deadline: Duration,
|
||||
) -> Vec<Card> {
|
||||
let cap = n.min(MB_MAX_PAGES as usize * 100);
|
||||
self.mb
|
||||
.release_groups_by_artist(artist_mbid, cap, deadline)
|
||||
.await
|
||||
.iter()
|
||||
.filter_map(|rg| self.album_card(rg))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn tag(name: &str, count: i64) -> Tag {
|
||||
Tag {
|
||||
name: name.to_string(),
|
||||
count,
|
||||
}
|
||||
}
|
||||
|
||||
/// Curated genres win outright — a curated "Jazz" must not be diluted by
|
||||
/// whatever the tag crowd also wrote.
|
||||
#[test]
|
||||
fn curated_genres_beat_open_tags() {
|
||||
let got = genre_names(&[tag("jazz", 3)], &[tag("cool", 9), tag("stuff i like", 9)]);
|
||||
assert_eq!(got, vec!["Jazz"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tags_are_the_fallback_ranked_by_votes() {
|
||||
let got = genre_names(
|
||||
&[],
|
||||
&[tag("rock", 2), tag("hip hop", 7), tag("noise", 0)],
|
||||
);
|
||||
// Sorted by votes desc; the zero-vote tag is filtered out entirely.
|
||||
assert_eq!(got, vec!["Hip Hop", "Rock"]);
|
||||
}
|
||||
|
||||
/// Names join the same key-set other feeders write, so they carry the same
|
||||
/// display casing rather than MusicBrainz's lowercase convention.
|
||||
#[test]
|
||||
fn genre_names_are_title_cased() {
|
||||
assert_eq!(genre_names(&[tag("modal jazz", 4)], &[]), vec!["Modal Jazz"]);
|
||||
}
|
||||
|
||||
/// ⚠ Regression guard, seen live: splitting on spaces alone rendered the
|
||||
/// real MusicBrainz tag `r&b` as `R&b`. Since the name *is* the key-set
|
||||
/// member, that would put `genres/R&b` and `genres/R&B` on the same record
|
||||
/// as two different members of one set.
|
||||
#[test]
|
||||
fn capitalisation_happens_after_punctuation_too() {
|
||||
assert_eq!(title_case("r&b"), "R&B");
|
||||
assert_eq!(title_case("hip-hop"), "Hip-Hop");
|
||||
assert_eq!(title_case("drum'n'bass"), "Drum'N'Bass");
|
||||
assert_eq!(title_case("trip hop"), "Trip Hop");
|
||||
assert_eq!(title_case(""), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn genres_are_capped() {
|
||||
let many: Vec<Tag> = (0..20).map(|i| tag(&format!("g{i}"), 20 - i)).collect();
|
||||
assert_eq!(genre_names(&many, &[]).len(), MAX_GENRES);
|
||||
}
|
||||
|
||||
/// A ListenBrainz entry that already names a CAA image must use it — that
|
||||
/// image is known to exist, whereas the release-group front endpoint 404s
|
||||
/// silently and the card is then hidden downstream.
|
||||
#[test]
|
||||
fn a_popular_entry_prefers_its_resolved_cover() {
|
||||
let r = Resolver::new(
|
||||
Arc::new(MusicBrainzClient::with_base(
|
||||
"http://unused".into(),
|
||||
meta_feeder_sdk::budget::RateBudget::new(1.0, 1.0),
|
||||
open_test_cache(),
|
||||
)),
|
||||
Arc::new(ListenBrainzClient::with_base(
|
||||
"http://unused".into(),
|
||||
meta_feeder_sdk::budget::RateBudget::new(1.0, 1.0),
|
||||
)),
|
||||
);
|
||||
let with_caa = PopularWork {
|
||||
release_group_mbid: "rg".into(),
|
||||
title: "T".into(),
|
||||
artist_name: "A".into(),
|
||||
artist_mbid: Some("am".into()),
|
||||
caa: Some(("rel".into(), 7)),
|
||||
release_date: None,
|
||||
};
|
||||
let card = r.album_card_from_popular(&with_caa).expect("card");
|
||||
assert!(matches!(card.cover, Some(Cover::Resolved { .. })));
|
||||
|
||||
let without = PopularWork {
|
||||
caa: None,
|
||||
..with_caa
|
||||
};
|
||||
let card = r.album_card_from_popular(&without).expect("card");
|
||||
assert!(matches!(card.cover, Some(Cover::ReleaseGroupFront { .. })));
|
||||
}
|
||||
|
||||
fn open_test_cache() -> meta_feeder_sdk::cache::MidhashCache {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let c = meta_feeder_sdk::cache::MidhashCache::open(dir.path()).expect("open");
|
||||
// Leak the tempdir so the redb file outlives this call in-test.
|
||||
std::mem::forget(dir);
|
||||
c
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
[toolchain]
|
||||
channel = "1.89"
|
||||
components = ["rustfmt", "clippy"]
|
||||
Reference in New Issue
Block a user