Compare commits
5
Commits
v1.0.2
..
8bf94cb186
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8bf94cb186 | ||
|
|
9d20e3ae15 | ||
|
|
87df5ef104 | ||
|
|
e87afa68c8 | ||
|
|
aed0db3cda |
@@ -0,0 +1,85 @@
|
|||||||
|
# Gitea twin of .github/workflows/publish.yml. Gitea runs only .gitea/workflows/ once this
|
||||||
|
# folder exists, and GitHub never reads it.
|
||||||
|
#
|
||||||
|
# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
|
||||||
|
# push ipcr-push-85-17-246-67.nip.io/metadec.eth/<repo>:<tag> docker login as metadec.eth with the
|
||||||
|
# IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
|
||||||
|
# pull ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag> anonymous
|
||||||
|
# The push goes to ipcr-push-85-17-246-67.nip.io, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
|
||||||
|
# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
|
||||||
|
# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
|
||||||
|
# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
|
||||||
|
# certificate without that CA, so the login is written straight into ~/.docker/config.json.
|
||||||
|
# The runner's job image (node:22-bookworm) has no docker CLI, so the static one is
|
||||||
|
# installed first; buildx installs itself. No `type=gha` cache: the runner has none.
|
||||||
|
|
||||||
|
name: Publish image
|
||||||
|
|
||||||
|
# Build and push the feeder image to the IPCR (ipcr-push-watch.nsl.sh):
|
||||||
|
# * every push to main -> the moving `main` tag
|
||||||
|
# * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1)
|
||||||
|
# The build context is the repo root (the workspace); `meta-feeder-sdk` is a
|
||||||
|
# tag-pinned git dependency fetched during the build. The Dockerfile lives under
|
||||||
|
# feeder-plugin/book-feeder/.
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [main]
|
||||||
|
tags: ['v*']
|
||||||
|
|
||||||
|
env:
|
||||||
|
REGISTRY: ipcr-push-85-17-246-67.nip.io
|
||||||
|
IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
publish:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
packages: write
|
||||||
|
steps:
|
||||||
|
- name: Checkout
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- name: Install Docker CLI
|
||||||
|
run: curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-28.5.1.tgz | tar -xz -C /usr/local/bin --strip-components=1 docker/docker
|
||||||
|
|
||||||
|
- name: Trust the Mesh Router CA
|
||||||
|
run: printf '%s\n' "$IPCR_CA_PEM" > /tmp/ipcr-ca.pem
|
||||||
|
env:
|
||||||
|
IPCR_CA_PEM: ${{ vars.IPCR_CA_PEM }}
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
with:
|
||||||
|
version: latest
|
||||||
|
buildkitd-config-inline: |
|
||||||
|
[registry."${{ env.REGISTRY }}"]
|
||||||
|
ca = ["/tmp/ipcr-ca.pem"]
|
||||||
|
|
||||||
|
- name: Log in to IPCR
|
||||||
|
env:
|
||||||
|
IPCR_PUSH_TOKEN: ${{ secrets.IPCR_PUSH_TOKEN }}
|
||||||
|
run: |
|
||||||
|
mkdir -p ~/.docker
|
||||||
|
printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY" "$(printf 'metadec.eth:%s' "$IPCR_PUSH_TOKEN" | base64 -w0)" > ~/.docker/config.json
|
||||||
|
|
||||||
|
- name: Extract image metadata
|
||||||
|
id: meta
|
||||||
|
uses: docker/metadata-action@v5
|
||||||
|
with:
|
||||||
|
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||||
|
tags: |
|
||||||
|
type=ref,event=branch
|
||||||
|
type=semver,pattern={{version}}
|
||||||
|
type=semver,pattern={{major}}.{{minor}}
|
||||||
|
type=semver,pattern={{major}}
|
||||||
|
type=sha,format=short
|
||||||
|
|
||||||
|
- name: Build and push
|
||||||
|
uses: docker/build-push-action@v6
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: ./feeder-plugin/book-feeder/Dockerfile
|
||||||
|
push: true
|
||||||
|
tags: ${{ steps.meta.outputs.tags }}
|
||||||
|
labels: ${{ steps.meta.outputs.labels }}
|
||||||
@@ -3,8 +3,8 @@ name: Publish image
|
|||||||
# Build and push the feeder image to the GitHub Container Registry (ghcr.io):
|
# Build and push the feeder image to the GitHub Container Registry (ghcr.io):
|
||||||
# * every push to main -> the moving `main` tag
|
# * every push to main -> the moving `main` tag
|
||||||
# * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1)
|
# * every `v*` tag -> semver tags (e.g. v1.0.0 -> 1.0.0, 1.0, 1)
|
||||||
# The build context is the repo root (the workspace) so the vendored
|
# The build context is the repo root (the workspace); `meta-feeder-sdk` is a
|
||||||
# `meta-feeder-sdk` path dependency resolves; the Dockerfile lives under
|
# tag-pinned git dependency fetched during the build. The Dockerfile lives under
|
||||||
# feeder-plugin/book-feeder/.
|
# feeder-plugin/book-feeder/.
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
|
|||||||
Generated
+3
-2
@@ -791,7 +791,8 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "meta-feeder-sdk"
|
name = "meta-feeder-sdk"
|
||||||
version = "0.1.0"
|
version = "1.7.0"
|
||||||
|
source = "git+https://github.com/worph/meta-feeder-sdk?tag=v1.7.0#027917c8cffe7ca43dadcf3b426cc5f69ecc443a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
@@ -800,13 +801,13 @@ dependencies = [
|
|||||||
"bytes",
|
"bytes",
|
||||||
"futures",
|
"futures",
|
||||||
"hyper",
|
"hyper",
|
||||||
|
"libc",
|
||||||
"redb",
|
"redb",
|
||||||
"regex",
|
"regex",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2",
|
"sha2",
|
||||||
"tempfile",
|
|
||||||
"thiserror 1.0.69",
|
"thiserror 1.0.69",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tracing",
|
"tracing",
|
||||||
|
|||||||
+5
-1
@@ -1,6 +1,10 @@
|
|||||||
[workspace]
|
[workspace]
|
||||||
resolver = "2"
|
resolver = "2"
|
||||||
members = ["crates/*", "feeder-plugin/*"]
|
members = ["feeder-plugin/*"]
|
||||||
|
# The vendored SDK copy is no longer built: book-feeder pins meta-feeder-sdk by
|
||||||
|
# git tag. Excluded (not deleted) so a stray `cargo` run inside it does not try
|
||||||
|
# to join this workspace.
|
||||||
|
exclude = ["crates/meta-feeder-sdk"]
|
||||||
|
|
||||||
[workspace.package]
|
[workspace.package]
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
|||||||
@@ -1,12 +1,23 @@
|
|||||||
# meta-feeder-book
|
# meta-feeder-book
|
||||||
|
|
||||||
Book **feeder sidecar** for [MetaMesh](https://github.com/worph/meta-gateway) —
|
Book **feeder sidecar** for [MetaMesh](https://github.com/worph/meta-gateway).
|
||||||
surfaces public-domain books from **Project Gutenberg** (via the
|
One binary, `book-feeder`, hosts two upstreams:
|
||||||
[Gutendex](https://gutendex.com) API) into a meta-gateway.
|
|
||||||
|
|
||||||
It is the proof-of-concept feeder: a single upstream, no preview-seeding, no
|
| Upstream | Serves | What it is |
|
||||||
BitTorrent — just enough to exercise the `meta-feeder-sdk` + feeder HTTP
|
|----------|--------|------------|
|
||||||
contract end to end.
|
| `gutenberg` | `document` × `book` | Public-domain **editions** from Project Gutenberg (via the [Gutendex](https://gutendex.com) API), with an epub url-locator per record |
|
||||||
|
| `openlibrary` | `card` × `book` | **Work cards** from [Open Library](https://openlibrary.org) — the book identity tier (`TMDB : video :: AniList : comics :: Open Library : books`) |
|
||||||
|
|
||||||
|
The two meet through `openlibraryid`. A Gutenberg edition carries it only when
|
||||||
|
Open Library itself records the link: exactly one Open Library work lists the
|
||||||
|
etext's Project Gutenberg id (`search.json?q=id_project_gutenberg:<id>`). The
|
||||||
|
edition then also carries the transient trust claim `anchored=true`,
|
||||||
|
`anchorSource=openlibrary`, which the gateway stores as
|
||||||
|
`anchoredBy/openlibrary:OL…W`. There is no title or author matching.
|
||||||
|
|
||||||
|
An Open Library card is addressed `compute_card_cid("openlibrary", "OL…W")`,
|
||||||
|
the bare work id, byte-identical to meta-read's `card_cid::openlibrary`. A work
|
||||||
|
with no cover or no description gets no card.
|
||||||
|
|
||||||
## Role in MetaMesh
|
## Role in MetaMesh
|
||||||
|
|
||||||
@@ -32,9 +43,16 @@ drives the contract:
|
|||||||
|---------|---------|-------|
|
|---------|---------|-------|
|
||||||
| `META_FEEDER_HTTP_LISTEN` | `0.0.0.0:8080` | HTTP listen address |
|
| `META_FEEDER_HTTP_LISTEN` | `0.0.0.0:8080` | HTTP listen address |
|
||||||
| `META_FEEDER_STATE_DIR` | `/data/meta-feeder` | redb cache + state |
|
| `META_FEEDER_STATE_DIR` | `/data/meta-feeder` | redb cache + state |
|
||||||
|
| `OPENLIBRARY_CONTACT` | — | e-mail or URL sent in the Open Library User-Agent. First-boot seed for the `contact` config field; a value saved in the config UI wins |
|
||||||
|
| `OPENLIBRARY_TOP_N` | `12` | cards per search |
|
||||||
|
| `OPENLIBRARY_DISCOVERY_N` | `20` | cards per browse row (`popular:true` → Open Library trending, `fresh:true` → recently added) |
|
||||||
| `RUST_LOG` | `info` | tracing filter |
|
| `RUST_LOG` | `info` | tracing filter |
|
||||||
|
|
||||||
No upstream API key is required — Gutendex is open.
|
No upstream API key is required. Open Library asks API clients to identify
|
||||||
|
themselves, so set a contact in production: an identified client gets a larger
|
||||||
|
request budget. Both upstreams share one Open Library rate budget. The config
|
||||||
|
page edits the `gutenberg` upstream, and the contact saved there is read by
|
||||||
|
both.
|
||||||
|
|
||||||
## Image
|
## Image
|
||||||
|
|
||||||
@@ -47,24 +65,32 @@ tag) and on `v*` tags (semver tags).
|
|||||||
|
|
||||||
## Build locally
|
## Build locally
|
||||||
|
|
||||||
The build context is the **repo root** (the Cargo workspace) so the vendored
|
The build context is the **repo root** (the Cargo workspace).
|
||||||
`meta-feeder-sdk` path dependency resolves:
|
`meta-feeder-sdk` is a git dependency pinned by tag, so the builder needs
|
||||||
|
network access to github.com:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker build -f feeder-plugin/book-feeder/Dockerfile -t ghcr.io/worph/meta-feeder-book:dev .
|
docker build -f feeder-plugin/book-feeder/Dockerfile -t ghcr.io/worph/meta-feeder-book:dev .
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Tests (`cargo` is not needed on the host; wiremock stands in for every upstream):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --rm -v "$PWD":/src -w /src -e CARGO_TARGET_DIR=/target rust:1.89 \
|
||||||
|
cargo test -p book-feeder
|
||||||
|
```
|
||||||
|
|
||||||
## Repo layout
|
## Repo layout
|
||||||
|
|
||||||
This repo is a self-contained Cargo workspace vendored out of the
|
|
||||||
`meta-gateway` monorepo:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
Cargo.toml # workspace: members = crates/*, feeder-plugin/*
|
Cargo.toml # workspace: members = feeder-plugin/*
|
||||||
crates/meta-feeder-sdk/ # vendored shared feeder SDK
|
|
||||||
feeder-plugin/book-feeder/ # this feeder's crate + Dockerfile
|
feeder-plugin/book-feeder/ # this feeder's crate + Dockerfile
|
||||||
|
src/gutenberg.rs # `gutenberg` upstream
|
||||||
|
src/openlibrary.rs # `openlibrary` upstream (cards)
|
||||||
|
src/openlibrary_client.rs # Open Library client (budget, cache, binding lookup)
|
||||||
|
crates/meta-feeder-sdk/ # old vendored SDK copy — no longer built (excluded)
|
||||||
```
|
```
|
||||||
|
|
||||||
Upstream source of truth for the SDK and the feeder crate is
|
`meta-feeder-sdk` comes from [`worph/meta-feeder-sdk`](https://github.com/worph/meta-feeder-sdk)
|
||||||
[`worph/meta-gateway`](https://github.com/worph/meta-gateway); changes there are
|
at the tag in `feeder-plugin/book-feeder/Cargo.toml`. An SDK change reaches
|
||||||
vendored back into this repo.
|
this feeder only when it is tagged and the pin is bumped.
|
||||||
|
|||||||
@@ -6,9 +6,10 @@ rust-version.workspace = true
|
|||||||
license.workspace = true
|
license.workspace = true
|
||||||
publish = false
|
publish = false
|
||||||
|
|
||||||
# Book feeder sidecar: Project Gutenberg (via Gutendex). The proof-of-concept
|
# Book feeder sidecar, two upstreams:
|
||||||
# feeder — single upstream, no preview-seeding, no BitTorrent — that validates
|
# - `gutenberg` — Project Gutenberg editions (via Gutendex), the byte tier.
|
||||||
# the meta-feeder-sdk + feeder HTTP contract end to end.
|
# - `openlibrary` — Open Library work cards, the book identity tier
|
||||||
|
# (`TMDB : video :: AniList : comics :: Open Library : books`).
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
name = "book_feeder"
|
name = "book_feeder"
|
||||||
@@ -19,7 +20,11 @@ name = "book-feeder"
|
|||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
meta-feeder-sdk = { path = "../../crates/meta-feeder-sdk" }
|
# Pinned by TAG, never by path. A path dep resolves OUTSIDE the crate, and the
|
||||||
|
# vendored copy this used to point at had already drifted behind the shared SDK
|
||||||
|
# (no `RateBudget`, a capped card cid). The rest of the fleet pins the same
|
||||||
|
# way; see meta-feeder-anilist's manifest.
|
||||||
|
meta-feeder-sdk = { git = "https://github.com/worph/meta-feeder-sdk", tag = "v1.7.0" }
|
||||||
|
|
||||||
tokio = { version = "1.40", features = ["full"] }
|
tokio = { version = "1.40", features = ["full"] }
|
||||||
axum = { version = "0.7", features = ["json"] }
|
axum = { version = "0.7", features = ["json"] }
|
||||||
|
|||||||
@@ -1,12 +1,13 @@
|
|||||||
# syntax=docker/dockerfile:1
|
# syntax=docker/dockerfile:1
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# book-feeder — Project Gutenberg feeder sidecar. Multi-stage build.
|
# book-feeder — Project Gutenberg + Open Library feeder sidecar. Multi-stage
|
||||||
|
# build; build context is this repo's root (the Cargo workspace).
|
||||||
#
|
#
|
||||||
# The build context must be `packages/meta-gateway/` (the workspace root) so
|
# meta-feeder-sdk arrives as a git dependency (tag-pinned), so the vendored
|
||||||
# the `meta-feeder-sdk` path dependency (../../crates/meta-feeder-sdk) resolves.
|
# `crates/` copy is not copied in — `git` must be present in the builder for
|
||||||
# Same BuildKit cache-mount strategy as the gateway Dockerfile: the cargo
|
# cargo to fetch it. Same BuildKit cache-mount strategy as the gateway
|
||||||
# registry/git and `target/` persist across builds. The release binary is
|
# Dockerfile: the cargo registry/git and `target/` persist across builds. The
|
||||||
# copied OUT of the target cache mount within the same RUN.
|
# release binary is copied OUT of the target cache mount within the same RUN.
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
||||||
FROM rust:1.89-slim-bookworm AS builder
|
FROM rust:1.89-slim-bookworm AS builder
|
||||||
@@ -20,7 +21,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
COPY Cargo.toml Cargo.lock rust-toolchain.toml ./
|
||||||
COPY crates/ crates/
|
|
||||||
COPY feeder-plugin/ feeder-plugin/
|
COPY feeder-plugin/ feeder-plugin/
|
||||||
|
|
||||||
RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \
|
RUN --mount=type=cache,id=cargo-registry,target=/usr/local/cargo/registry \
|
||||||
|
|||||||
@@ -6,17 +6,24 @@
|
|||||||
//! don't expose an epub variant fail `compute_outcomes` with a permanent
|
//! don't expose an epub variant fail `compute_outcomes` with a permanent
|
||||||
//! error (operator-visible; very rare in practice).
|
//! error (operator-visible; very rare in practice).
|
||||||
//!
|
//!
|
||||||
//! No auth, no API key, no rate-limit configuration. Moved verbatim from the
|
//! - Per-edition metadata: `GET {rdf_base}/cache/epub/{id}/pg{id}.rdf` for the
|
||||||
//! gateway crate's `plugins/gutenberg.rs`; only the import paths changed
|
//! fields Gutendex does not carry (summary, release date, credits).
|
||||||
//! (`crate::*`/`super::common` → `meta_feeder_sdk::*`, `GatewayPlugin` →
|
//! - Work binding: Open Library's `search.json?q=id_project_gutenberg:{id}` —
|
||||||
//! `FeederPlugin`).
|
//! see [`OpenLibraryClient::gutenberg_work`] and reading-model §2.4.
|
||||||
|
//!
|
||||||
|
//! No auth, no API key. Originally moved verbatim from the gateway crate's
|
||||||
|
//! `plugins/gutenberg.rs`.
|
||||||
|
|
||||||
use std::collections::{BTreeMap, HashMap};
|
use std::collections::{BTreeMap, HashMap};
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
use meta_feeder_sdk::budget::RateBudget;
|
||||||
use meta_feeder_sdk::cache::MidhashCache;
|
use meta_feeder_sdk::cache::MidhashCache;
|
||||||
use meta_feeder_sdk::common;
|
use meta_feeder_sdk::common;
|
||||||
|
use meta_feeder_sdk::config::ConfigSchema;
|
||||||
|
use meta_feeder_sdk::lang::normalize_lang_code;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use meta_feeder_sdk::plugin::HashKind;
|
use meta_feeder_sdk::plugin::HashKind;
|
||||||
use meta_feeder_sdk::plugin::{upstream_id_field, ConfigError, FeederPlugin, HashOutcome};
|
use meta_feeder_sdk::plugin::{upstream_id_field, ConfigError, FeederPlugin, HashOutcome};
|
||||||
@@ -25,10 +32,22 @@ use meta_feeder_sdk::types::{DiscoveryRecord, GatewayError, PluginHealth};
|
|||||||
use serde::Deserialize;
|
use serde::Deserialize;
|
||||||
use tracing::{debug, warn};
|
use tracing::{debug, warn};
|
||||||
|
|
||||||
|
use crate::openlibrary_client::{self, OpenLibraryClient};
|
||||||
|
|
||||||
/// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`]
|
/// Canonical Gutendex base URL. Overridable via [`GutenbergPlugin::with_base_url`]
|
||||||
/// for tests + private mirrors.
|
/// for tests + private mirrors.
|
||||||
const DEFAULT_BASE_URL: &str = "https://gutendex.com";
|
const DEFAULT_BASE_URL: &str = "https://gutendex.com";
|
||||||
|
|
||||||
|
/// Canonical host for the per-edition RDF. Overridable via
|
||||||
|
/// [`GutenbergPlugin::with_rdf_base_url`] — until it was, every unit test that
|
||||||
|
/// ran a query also fetched live RDF from gutenberg.org.
|
||||||
|
const DEFAULT_RDF_BASE_URL: &str = "https://www.gutenberg.org";
|
||||||
|
|
||||||
|
/// Upper bound on one binding lookup on the live search path, on top of the
|
||||||
|
/// budget deadline. A slow Open Library must not hold a Gutenberg result that
|
||||||
|
/// is already in hand.
|
||||||
|
const BINDING_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(8);
|
||||||
|
|
||||||
/// Format we treat as the canonical hashable bytes for a Gutenberg record.
|
/// Format we treat as the canonical hashable bytes for a Gutenberg record.
|
||||||
const EPUB_FORMAT: &str = "application/epub+zip";
|
const EPUB_FORMAT: &str = "application/epub+zip";
|
||||||
|
|
||||||
@@ -64,7 +83,18 @@ const RETRY_BACKOFF_MS: u64 = 250;
|
|||||||
pub struct GutenbergPlugin {
|
pub struct GutenbergPlugin {
|
||||||
http: reqwest::Client,
|
http: reqwest::Client,
|
||||||
base_url: String,
|
base_url: String,
|
||||||
|
rdf_base_url: String,
|
||||||
cache: Option<MidhashCache>,
|
cache: Option<MidhashCache>,
|
||||||
|
/// The Open Library binding client, built at `configure()`. `None` before
|
||||||
|
/// that — records are then simply unbound.
|
||||||
|
openlibrary: Option<OpenLibraryClient>,
|
||||||
|
/// Test hook for the Open Library API base. Never set in production.
|
||||||
|
openlibrary_base: Option<String>,
|
||||||
|
/// Shared with the `openlibrary` plugin by `main.rs`; built at
|
||||||
|
/// `configure()` when absent.
|
||||||
|
openlibrary_budget: Option<Arc<RateBudget>>,
|
||||||
|
/// The Open Library contact (see `openlibrary_client::load_contact`).
|
||||||
|
contact: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl GutenbergPlugin {
|
impl GutenbergPlugin {
|
||||||
@@ -80,10 +110,33 @@ impl GutenbergPlugin {
|
|||||||
Self {
|
Self {
|
||||||
http,
|
http,
|
||||||
base_url,
|
base_url,
|
||||||
|
rdf_base_url: DEFAULT_RDF_BASE_URL.to_string(),
|
||||||
cache: None,
|
cache: None,
|
||||||
|
openlibrary: None,
|
||||||
|
openlibrary_base: None,
|
||||||
|
openlibrary_budget: None,
|
||||||
|
contact: openlibrary_client::env_contact(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Fetch the per-edition RDF from `rdf_base_url` instead of gutenberg.org.
|
||||||
|
pub fn with_rdf_base_url(mut self, rdf_base_url: String) -> Self {
|
||||||
|
self.rdf_base_url = rdf_base_url;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Send the Open Library binding lookup to `base` — tests only.
|
||||||
|
pub fn with_openlibrary_base_url(mut self, base: String) -> Self {
|
||||||
|
self.openlibrary_base = Some(base);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Draw binding lookups from a budget shared with the `openlibrary` plugin.
|
||||||
|
pub fn with_openlibrary_budget(mut self, budget: Arc<RateBudget>) -> Self {
|
||||||
|
self.openlibrary_budget = Some(budget);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
fn cache(&self) -> Result<&MidhashCache, GatewayError> {
|
fn cache(&self) -> Result<&MidhashCache, GatewayError> {
|
||||||
common::require_cache(self.cache.as_ref(), "gutenberg")
|
common::require_cache(self.cache.as_ref(), "gutenberg")
|
||||||
}
|
}
|
||||||
@@ -151,11 +204,13 @@ impl GutenbergPlugin {
|
|||||||
const BIBREC_CONCURRENCY: usize = 6;
|
const BIBREC_CONCURRENCY: usize = 6;
|
||||||
let cache = self.cache.clone();
|
let cache = self.cache.clone();
|
||||||
let http = self.http.clone();
|
let http = self.http.clone();
|
||||||
|
let rdf_base = self.rdf_base_url.trim_end_matches('/').to_string();
|
||||||
let ids: Vec<u64> = books.iter().map(|b| b.id).collect();
|
let ids: Vec<u64> = books.iter().map(|b| b.id).collect();
|
||||||
stream::iter(ids.into_iter())
|
stream::iter(ids.into_iter())
|
||||||
.map(|id| {
|
.map(|id| {
|
||||||
let cache = cache.clone();
|
let cache = cache.clone();
|
||||||
let http = http.clone();
|
let http = http.clone();
|
||||||
|
let rdf_base = rdf_base.clone();
|
||||||
async move {
|
async move {
|
||||||
let record_id = id.to_string();
|
let record_id = id.to_string();
|
||||||
if let Some(cached) = cache
|
if let Some(cached) = cache
|
||||||
@@ -164,7 +219,7 @@ impl GutenbergPlugin {
|
|||||||
{
|
{
|
||||||
return cached;
|
return cached;
|
||||||
}
|
}
|
||||||
let url = format!("https://www.gutenberg.org/cache/epub/{id}/pg{id}.rdf");
|
let url = format!("{rdf_base}/cache/epub/{id}/pg{id}.rdf");
|
||||||
let resp = tokio::time::timeout(
|
let resp = tokio::time::timeout(
|
||||||
std::time::Duration::from_secs(8),
|
std::time::Duration::from_secs(8),
|
||||||
http.get(&url).send(),
|
http.get(&url).send(),
|
||||||
@@ -334,6 +389,57 @@ impl GutenbergPlugin {
|
|||||||
.collect()
|
.collect()
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Each result's Open Library work, through the Project Gutenberg id Open
|
||||||
|
/// Library records (reading-model §2.4). Aligned with the input slice;
|
||||||
|
/// `None` means no binding — none recorded, ambiguous, not configured, over
|
||||||
|
/// budget, or the lookup failed.
|
||||||
|
///
|
||||||
|
/// ⚠ **A failure never blocks or drops a record.** The binding upgrades an
|
||||||
|
/// edition that is already readable. Every failure is left uncached, so the
|
||||||
|
/// next query that surfaces the book tries again; the budget deadline is
|
||||||
|
/// short on purpose (`BINDING_DEADLINE`), so a cold browse row binds the
|
||||||
|
/// first handful of books and the cache fills the rest over later queries.
|
||||||
|
async fn fetch_openlibrary_works(&self, books: &[GutendexBook]) -> Vec<Option<String>> {
|
||||||
|
use futures::stream::{self, StreamExt};
|
||||||
|
const BINDING_CONCURRENCY: usize = 4;
|
||||||
|
let Some(client) = self.openlibrary.as_ref() else {
|
||||||
|
return vec![None; books.len()];
|
||||||
|
};
|
||||||
|
// Owned ids, not `books.iter().map(..)`: a borrowing closure inside the
|
||||||
|
// stream makes `handle_query`'s future fail the `async_trait` lifetime
|
||||||
|
// check ("FnOnce is not general enough").
|
||||||
|
let ids: Vec<u64> = books.iter().map(|b| b.id).collect();
|
||||||
|
stream::iter(ids)
|
||||||
|
.map(|id| async move {
|
||||||
|
let lookup = client.gutenberg_work(id, openlibrary_client::BINDING_DEADLINE);
|
||||||
|
match tokio::time::timeout(BINDING_TIMEOUT, lookup).await {
|
||||||
|
Ok(Ok(work)) => work,
|
||||||
|
Ok(Err(e)) => {
|
||||||
|
debug!(
|
||||||
|
target: "meta-share::gateway",
|
||||||
|
upstream = "gutenberg",
|
||||||
|
id,
|
||||||
|
error = %e,
|
||||||
|
"open library binding lookup failed; record left unbound"
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
debug!(
|
||||||
|
target: "meta-share::gateway",
|
||||||
|
upstream = "gutenberg",
|
||||||
|
id,
|
||||||
|
"open library binding lookup timed out; record left unbound"
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.buffered(BINDING_CONCURRENCY)
|
||||||
|
.collect()
|
||||||
|
.await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for GutenbergPlugin {
|
impl Default for GutenbergPlugin {
|
||||||
@@ -349,7 +455,22 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> {
|
fn configure(&mut self, cache_dir: &Path) -> Result<(), ConfigError> {
|
||||||
self.cache = Some(common::open_midhash_cache(cache_dir, "gutenberg")?);
|
let cache = common::open_midhash_cache(cache_dir, "gutenberg")?;
|
||||||
|
// The binding cache rides this plugin's own redb (the `misc` table), so
|
||||||
|
// a binding survives a restart without touching the card plugin's store.
|
||||||
|
let contact = openlibrary_client::load_contact(cache_dir);
|
||||||
|
let budget = self
|
||||||
|
.openlibrary_budget
|
||||||
|
.clone()
|
||||||
|
.unwrap_or_else(|| openlibrary_client::budget(contact.is_some()));
|
||||||
|
self.openlibrary = Some(OpenLibraryClient::new(
|
||||||
|
budget,
|
||||||
|
Some(cache.clone()),
|
||||||
|
self.openlibrary_base.clone(),
|
||||||
|
contact.as_deref(),
|
||||||
|
));
|
||||||
|
self.contact = contact;
|
||||||
|
self.cache = Some(cache);
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -405,13 +526,19 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
"search returned"
|
"search returned"
|
||||||
);
|
);
|
||||||
let raw: Vec<GutendexBook> = body.results.into_iter().take(max_results).collect();
|
let raw: Vec<GutendexBook> = body.results.into_iter().take(max_results).collect();
|
||||||
let (cover_cids, bibrecs) =
|
let (cover_cids, bibrecs, works) = tokio::join!(
|
||||||
tokio::join!(self.fetch_cover_cids(&raw), self.fetch_bibrec_fields(&raw),);
|
self.fetch_cover_cids(&raw),
|
||||||
|
self.fetch_bibrec_fields(&raw),
|
||||||
|
self.fetch_openlibrary_works(&raw),
|
||||||
|
);
|
||||||
Ok(raw
|
Ok(raw
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.zip(cover_cids)
|
.zip(cover_cids)
|
||||||
.zip(bibrecs)
|
.zip(bibrecs)
|
||||||
.map(|((book, cover_cid), bibrec)| into_discovery_record(book, cover_cid, bibrec))
|
.zip(works)
|
||||||
|
.map(|(((book, cover_cid), bibrec), work)| {
|
||||||
|
into_discovery_record(book, cover_cid, bibrec, work)
|
||||||
|
})
|
||||||
.collect())
|
.collect())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -438,7 +565,26 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
.ok()
|
.ok()
|
||||||
.flatten()
|
.flatten()
|
||||||
.unwrap_or_default();
|
.unwrap_or_default();
|
||||||
let record = into_discovery_record(book, cover_cid, bibrec);
|
// Not user-facing, so the binding lookup may queue for the full search
|
||||||
|
// deadline. A failure still leaves the record unbound, never fails the
|
||||||
|
// compute.
|
||||||
|
let work = match self.openlibrary.as_ref() {
|
||||||
|
Some(ol) => ol
|
||||||
|
.gutenberg_work(book.id, openlibrary_client::SEARCH_DEADLINE)
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|e| {
|
||||||
|
debug!(
|
||||||
|
target: "meta-share::gateway",
|
||||||
|
upstream = "gutenberg",
|
||||||
|
record_id,
|
||||||
|
error = %e,
|
||||||
|
"open library binding lookup failed; record left unbound"
|
||||||
|
);
|
||||||
|
None
|
||||||
|
}),
|
||||||
|
None => None,
|
||||||
|
};
|
||||||
|
let record = into_discovery_record(book, cover_cid, bibrec, work);
|
||||||
|
|
||||||
Ok(common::single_outcome(
|
Ok(common::single_outcome(
|
||||||
cid,
|
cid,
|
||||||
@@ -481,13 +627,30 @@ impl FeederPlugin for GutenbergPlugin {
|
|||||||
fn served_content_kinds(&self) -> &'static [&'static str] {
|
fn served_content_kinds(&self) -> &'static [&'static str] {
|
||||||
&["book"]
|
&["book"]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// ⚠ Declared HERE, not only on `openlibrary`. The SDK's `/config` page
|
||||||
|
/// edits the lowest hosted upstream id, which is this one — so this is where
|
||||||
|
/// the operator sets the contact both upstreams send to Open Library.
|
||||||
|
fn config_schema(&self) -> ConfigSchema {
|
||||||
|
ConfigSchema {
|
||||||
|
fields: vec![openlibrary_client::contact_field()],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn config_values(&self) -> serde_json::Value {
|
||||||
|
serde_json::json!({ "contact": self.contact.clone().unwrap_or_default() })
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Convert a `GutendexBook` into our wire-level `DiscoveryRecord`.
|
/// Convert a `GutendexBook` into our wire-level `DiscoveryRecord`.
|
||||||
|
///
|
||||||
|
/// `openlibrary_work` is the binding from [`OpenLibraryClient::gutenberg_work`]:
|
||||||
|
/// `Some` only when Open Library records exactly one work for this etext.
|
||||||
fn into_discovery_record(
|
fn into_discovery_record(
|
||||||
book: GutendexBook,
|
book: GutendexBook,
|
||||||
cover_cid: Option<String>,
|
cover_cid: Option<String>,
|
||||||
bibrec: BTreeMap<String, String>,
|
bibrec: BTreeMap<String, String>,
|
||||||
|
openlibrary_work: Option<String>,
|
||||||
) -> DiscoveryRecord {
|
) -> DiscoveryRecord {
|
||||||
let GutendexBook {
|
let GutendexBook {
|
||||||
id,
|
id,
|
||||||
@@ -520,6 +683,8 @@ fn into_discovery_record(
|
|||||||
fields.insert(upstream_id_field("gutenberg"), id.to_string());
|
fields.insert(upstream_id_field("gutenberg"), id.to_string());
|
||||||
|
|
||||||
if !authors.is_empty() {
|
if !authors.is_empty() {
|
||||||
|
// Legacy scalar, kept verbatim for existing readers: Gutenberg's own
|
||||||
|
// "Last, First" spelling, comma-joined.
|
||||||
fields.insert(
|
fields.insert(
|
||||||
"author".into(),
|
"author".into(),
|
||||||
authors
|
authors
|
||||||
@@ -529,9 +694,28 @@ fn into_discovery_record(
|
|||||||
.join(", "),
|
.join(", "),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
// `authors/{name}` key-set (reading-model §3.2): one member per person, in
|
||||||
|
// display order, so one author compares equal across sources — Open Library
|
||||||
|
// and AniList both write "Jane Austen", never "Austen, Jane".
|
||||||
|
for a in &authors {
|
||||||
|
if let Some(name) = display_name(&a.name) {
|
||||||
|
fields.insert(format!("authors/{name}"), "true".into());
|
||||||
|
}
|
||||||
|
}
|
||||||
if let Some(lang) = languages.first() {
|
if let Some(lang) = languages.first() {
|
||||||
fields.insert("language".into(), iso639_1_to_2(lang).to_string());
|
fields.insert("language".into(), iso639_1_to_2(lang).to_string());
|
||||||
}
|
}
|
||||||
|
// `languages/{lang3}` key-set for EVERY language, in the stack's shared
|
||||||
|
// vocabulary (`fre`, where the legacy scalar above says `fra`). Two-letter
|
||||||
|
// codes only the local table knows (`la`, `ca`, `is`) are widened first;
|
||||||
|
// anything still not three letters is dropped rather than written as a
|
||||||
|
// member no `languages:` filter can match.
|
||||||
|
for lang in &languages {
|
||||||
|
let code = normalize_lang_code(iso639_1_to_2(&lang.trim().to_ascii_lowercase()));
|
||||||
|
if code.len() == 3 && code.bytes().all(|b| b.is_ascii_lowercase()) {
|
||||||
|
fields.insert(format!("languages/{code}"), "true".into());
|
||||||
|
}
|
||||||
|
}
|
||||||
if !subjects.is_empty() {
|
if !subjects.is_empty() {
|
||||||
fields.insert("subjects".into(), subjects.join(", "));
|
fields.insert("subjects".into(), subjects.join(", "));
|
||||||
}
|
}
|
||||||
@@ -619,10 +803,37 @@ fn into_discovery_record(
|
|||||||
let _ = download_count;
|
let _ = download_count;
|
||||||
fields.insert("fileName".into(), format!("gutenberg-{id}.epub"));
|
fields.insert("fileName".into(), format!("gutenberg-{id}.epub"));
|
||||||
|
|
||||||
|
// The work binding and its trust claim (reading-model §2.4). Written only on
|
||||||
|
// an identifier link Open Library itself records, so the claim is the plain
|
||||||
|
// Anchored form: `anchored` + `anchorSource`, and deliberately NO
|
||||||
|
// `anchorMethod` — that marks a title match, which this never is. The
|
||||||
|
// gateway transcribes the claim into `anchoredBy/openlibrary:OL…W` and
|
||||||
|
// strips the transient keys before storing.
|
||||||
|
if let Some(work) = openlibrary_work.filter(|w| openlibrary_client::is_work_id(w)) {
|
||||||
|
fields.insert("openlibraryid".into(), work);
|
||||||
|
fields.insert("anchored".into(), "true".into());
|
||||||
|
fields.insert("anchorSource".into(), "openlibrary".into());
|
||||||
|
}
|
||||||
|
|
||||||
for (k, v) in bibrec {
|
for (k, v) in bibrec {
|
||||||
fields.entry(k).or_insert(v);
|
fields.entry(k).or_insert(v);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Registry spellings beside the legacy RDF keys, derived after the merge so
|
||||||
|
// a bibrec cached before these keys existed produces them too.
|
||||||
|
//
|
||||||
|
// `description/eng` from `pgterms:marc520` (kept as `summary`): Gutenberg's
|
||||||
|
// summaries are written in English whatever the book's language, and
|
||||||
|
// `description/{lang3}` is the form every card and client reads.
|
||||||
|
if let Some(summary) = fields.get("summary").cloned() {
|
||||||
|
fields.entry("description/eng".into()).or_insert(summary);
|
||||||
|
}
|
||||||
|
// `releasedate` (iso-date) beside the legacy camelCase `releaseDate`, only
|
||||||
|
// when the RDF value really is one.
|
||||||
|
if let Some(date) = fields.get("releaseDate").and_then(|d| iso_date(d)) {
|
||||||
|
fields.entry("releasedate".into()).or_insert(date);
|
||||||
|
}
|
||||||
|
|
||||||
DiscoveryRecord {
|
DiscoveryRecord {
|
||||||
upstream_id: "gutenberg".into(),
|
upstream_id: "gutenberg".into(),
|
||||||
record_id,
|
record_id,
|
||||||
@@ -630,6 +841,38 @@ fn into_discovery_record(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Gutenberg's "Last, First" → "First Last", the display order every other
|
||||||
|
/// writer of `authors/{name}` uses.
|
||||||
|
///
|
||||||
|
/// Only when there is **exactly one** comma. "King, Martin Luther, Jr." or a
|
||||||
|
/// corporate name has no safe reordering, and a wrong one is worse than
|
||||||
|
/// Gutenberg's own spelling, which is kept as is.
|
||||||
|
fn display_name(raw: &str) -> Option<String> {
|
||||||
|
let name = raw.trim();
|
||||||
|
if name.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
match name.split(',').map(str::trim).collect::<Vec<_>>().as_slice() {
|
||||||
|
[last, first] if !last.is_empty() && !first.is_empty() => Some(format!("{first} {last}")),
|
||||||
|
_ => Some(name.to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `raw` when it is an `iso-date` (`YYYY-MM-DD`, or a bare `YYYY`), else `None`.
|
||||||
|
///
|
||||||
|
/// Never padded: "1998" does not become "1998-01-01", which would claim a
|
||||||
|
/// precision the RDF never had.
|
||||||
|
fn iso_date(raw: &str) -> Option<String> {
|
||||||
|
let digits = |s: &str, n: usize| s.len() == n && s.bytes().all(|b| b.is_ascii_digit());
|
||||||
|
let t = raw.trim();
|
||||||
|
let ok = match t.split('-').collect::<Vec<_>>().as_slice() {
|
||||||
|
[y] => digits(y, 4),
|
||||||
|
[y, m, d] => digits(y, 4) && digits(m, 2) && digits(d, 2),
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
ok.then(|| t.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
/// Parse a Project Gutenberg `pg{id}.rdf` document for per-edition metadata.
|
/// Parse a Project Gutenberg `pg{id}.rdf` document for per-edition metadata.
|
||||||
fn parse_pgterms_rdf(xml: &str) -> BTreeMap<String, String> {
|
fn parse_pgterms_rdf(xml: &str) -> BTreeMap<String, String> {
|
||||||
let mut out = BTreeMap::new();
|
let mut out = BTreeMap::new();
|
||||||
@@ -762,6 +1005,28 @@ mod rdf_parser_tests {
|
|||||||
let f = parse_pgterms_rdf("<html><body>not rdf</body></html>");
|
let f = parse_pgterms_rdf("<html><body>not rdf</body></html>");
|
||||||
assert!(f.is_empty());
|
assert!(f.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// The legacy RDF keys stay; the registry spellings are derived beside them
|
||||||
|
/// on the record — including from a bibrec cached before those keys
|
||||||
|
/// existed, which holds only the legacy ones.
|
||||||
|
#[test]
|
||||||
|
fn summary_and_issued_date_also_arrive_under_registry_keys() {
|
||||||
|
let xml = r#"<rdf:RDF>
|
||||||
|
<dcterms:issued rdf:datatype="...">1998-06-01</dcterms:issued>
|
||||||
|
<pgterms:marc520>A classic novel of manners.</pgterms:marc520>
|
||||||
|
</rdf:RDF>"#;
|
||||||
|
let book: GutendexBook = serde_json::from_value(serde_json::json!({
|
||||||
|
"id": 1342,
|
||||||
|
"title": "Pride and Prejudice"
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
let r = into_discovery_record(book, None, parse_pgterms_rdf(xml), None);
|
||||||
|
let get = |k: &str| r.fields.get(k).map(String::as_str);
|
||||||
|
assert_eq!(get("summary"), Some("A classic novel of manners."));
|
||||||
|
assert_eq!(get("description/eng"), Some("A classic novel of manners."));
|
||||||
|
assert_eq!(get("releaseDate"), Some("1998-06-01"));
|
||||||
|
assert_eq!(get("releasedate"), Some("1998-06-01"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// ISO 639-1 (two-letter) → ISO 639-2/T (three-letter) for the ~20 most common
|
/// ISO 639-1 (two-letter) → ISO 639-2/T (three-letter) for the ~20 most common
|
||||||
@@ -877,13 +1142,27 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Every upstream — Gutendex, the RDF host and Open Library — on the one mock
|
||||||
|
/// server, so no test reaches the internet. An unmounted path 404s, which
|
||||||
|
/// every optional lookup treats as "nothing there".
|
||||||
fn configured_plugin_against(server: &MockServer) -> (GutenbergPlugin, tempfile::TempDir) {
|
fn configured_plugin_against(server: &MockServer) -> (GutenbergPlugin, tempfile::TempDir) {
|
||||||
let dir = tempfile::tempdir().expect("tempdir");
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
let mut plugin = GutenbergPlugin::with_base_url(server.uri());
|
let mut plugin = GutenbergPlugin::with_base_url(server.uri())
|
||||||
|
.with_rdf_base_url(server.uri())
|
||||||
|
.with_openlibrary_base_url(server.uri())
|
||||||
|
.with_openlibrary_budget(RateBudget::new(100.0, 100.0));
|
||||||
plugin.configure(dir.path()).expect("configure");
|
plugin.configure(dir.path()).expect("configure");
|
||||||
(plugin, dir)
|
(plugin, dir)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Open Library's answer to `q=id_project_gutenberg:<id>`.
|
||||||
|
fn binding(id: u64, body: serde_json::Value) -> Mock {
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("q", format!("id_project_gutenberg:{id}")))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(body))
|
||||||
|
}
|
||||||
|
|
||||||
/// ⚠ The search path must ask for the CANONICAL `/books/`.
|
/// ⚠ The search path must ask for the CANONICAL `/books/`.
|
||||||
///
|
///
|
||||||
/// Gutendex 301s `/books` → `/books/`. Following a redirect is free in
|
/// Gutendex 301s `/books` → `/books/`. Following a redirect is free in
|
||||||
@@ -1086,7 +1365,16 @@ mod tests {
|
|||||||
Some("https://www.gutenberg.org/ebooks/1342")
|
Some("https://www.gutenberg.org/ebooks/1342")
|
||||||
);
|
);
|
||||||
assert_eq!(r0.fields.get("author").map(String::as_str), Some("Austen, Jane"));
|
assert_eq!(r0.fields.get("author").map(String::as_str), Some("Austen, Jane"));
|
||||||
|
assert_eq!(
|
||||||
|
r0.fields.get("authors/Jane Austen").map(String::as_str),
|
||||||
|
Some("true")
|
||||||
|
);
|
||||||
|
assert!(!r0.fields.contains_key("authors/Austen, Jane"));
|
||||||
assert_eq!(r0.fields.get("language").map(String::as_str), Some("eng"));
|
assert_eq!(r0.fields.get("language").map(String::as_str), Some("eng"));
|
||||||
|
assert_eq!(r0.fields.get("languages/eng").map(String::as_str), Some("true"));
|
||||||
|
// No Open Library answer mounted: the record is returned, unbound.
|
||||||
|
assert!(!r0.fields.contains_key("openlibraryid"));
|
||||||
|
assert!(!r0.fields.contains_key("anchored"));
|
||||||
assert_eq!(r0.fields.get("format").map(String::as_str), Some("epub"));
|
assert_eq!(r0.fields.get("format").map(String::as_str), Some("epub"));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
r0.fields.get("fileName").map(String::as_str),
|
r0.fields.get("fileName").map(String::as_str),
|
||||||
@@ -1261,6 +1549,211 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn display_name_reorders_only_a_single_comma() {
|
||||||
|
assert_eq!(display_name("Austen, Jane").as_deref(), Some("Jane Austen"));
|
||||||
|
assert_eq!(
|
||||||
|
display_name("Shelley, Mary Wollstonecraft").as_deref(),
|
||||||
|
Some("Mary Wollstonecraft Shelley")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
display_name("King, Martin Luther, Jr.").as_deref(),
|
||||||
|
Some("King, Martin Luther, Jr.")
|
||||||
|
);
|
||||||
|
assert_eq!(display_name("Homer").as_deref(), Some("Homer"));
|
||||||
|
assert_eq!(display_name(" "), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn every_language_becomes_a_lang3_key_set_member() {
|
||||||
|
let book: GutendexBook = serde_json::from_value(serde_json::json!({
|
||||||
|
"id": 1,
|
||||||
|
"title": "Polyglot",
|
||||||
|
"languages": ["en", "fr", "la", "xx"]
|
||||||
|
}))
|
||||||
|
.unwrap();
|
||||||
|
let r = into_discovery_record(book, None, BTreeMap::new(), None);
|
||||||
|
let members: Vec<&str> = r
|
||||||
|
.fields
|
||||||
|
.keys()
|
||||||
|
.filter_map(|k| k.strip_prefix("languages/"))
|
||||||
|
.collect();
|
||||||
|
// The SDK vocabulary (`fre`), every language, and nothing that is not
|
||||||
|
// three letters.
|
||||||
|
assert_eq!(members, vec!["eng", "fre", "lat"]);
|
||||||
|
// The legacy scalar is untouched.
|
||||||
|
assert_eq!(r.fields.get("language").map(String::as_str), Some("eng"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn releasedate_is_written_only_for_an_iso_date() {
|
||||||
|
assert_eq!(iso_date("2008-06-27").as_deref(), Some("2008-06-27"));
|
||||||
|
assert_eq!(iso_date("1998").as_deref(), Some("1998"));
|
||||||
|
assert_eq!(iso_date("1998-06"), None);
|
||||||
|
assert_eq!(iso_date("June 1998"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The RDF host is configurable, so the per-edition fields are testable
|
||||||
|
/// without gutenberg.org.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn rdf_summary_and_issue_date_arrive_through_the_configured_rdf_host() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/books/"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/cache/epub/1342/pg1342.rdf"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_string(
|
||||||
|
r#"<rdf:RDF><pgterms:ebook rdf:about="ebooks/1342">
|
||||||
|
<dcterms:issued rdf:datatype="http://www.w3.org/2001/XMLSchema#date">1998-06-01</dcterms:issued>
|
||||||
|
<pgterms:marc520>"Pride and Prejudice" is a classic novel.</pgterms:marc520>
|
||||||
|
</pgterms:ebook></rdf:RDF>"#,
|
||||||
|
))
|
||||||
|
.expect(1)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let (plugin, _dir) = configured_plugin_against(&server);
|
||||||
|
let records = plugin
|
||||||
|
.handle_query(&GatewayQuery::from_free_text("alice"), 50)
|
||||||
|
.await
|
||||||
|
.expect("handle_query");
|
||||||
|
let pride = &records[0].fields;
|
||||||
|
assert!(pride["description/eng"].contains("classic novel"));
|
||||||
|
assert_eq!(pride.get("releasedate").map(String::as_str), Some("1998-06-01"));
|
||||||
|
// No RDF mounted for 84: neither key, and the record still arrives.
|
||||||
|
assert!(!records[1].fields.contains_key("description/eng"));
|
||||||
|
assert!(!records[1].fields.contains_key("releasedate"));
|
||||||
|
server.verify().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn exactly_one_open_library_work_binds_the_edition_with_an_anchored_claim() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/books/"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(
|
||||||
|
1342,
|
||||||
|
serde_json::json!({ "numFound": 1, "docs": [
|
||||||
|
{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342", "42671", "45186"] }
|
||||||
|
]}),
|
||||||
|
)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
// Two works list etext 84: an unmerged duplicate, so no binding at all.
|
||||||
|
binding(
|
||||||
|
84,
|
||||||
|
serde_json::json!({ "numFound": 2, "docs": [
|
||||||
|
{ "key": "/works/OL450063W", "id_project_gutenberg": ["84"] },
|
||||||
|
{ "key": "/works/OL18388W", "id_project_gutenberg": ["84"] }
|
||||||
|
]}),
|
||||||
|
)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let (plugin, _dir) = configured_plugin_against(&server);
|
||||||
|
let records = plugin
|
||||||
|
.handle_query(&GatewayQuery::from_free_text("alice"), 50)
|
||||||
|
.await
|
||||||
|
.expect("handle_query");
|
||||||
|
assert_eq!(records.len(), 2);
|
||||||
|
|
||||||
|
let pride = &records[0].fields;
|
||||||
|
assert_eq!(pride.get("openlibraryid").map(String::as_str), Some("OL66554W"));
|
||||||
|
assert_eq!(pride.get("anchored").map(String::as_str), Some("true"));
|
||||||
|
assert_eq!(pride.get("anchorSource").map(String::as_str), Some("openlibrary"));
|
||||||
|
assert!(
|
||||||
|
!pride.contains_key("anchorMethod"),
|
||||||
|
"an identifier link Open Library records is not a title match"
|
||||||
|
);
|
||||||
|
|
||||||
|
let frankenstein = &records[1].fields;
|
||||||
|
for k in ["openlibraryid", "anchored", "anchorSource", "anchorMethod"] {
|
||||||
|
assert!(!frankenstein.contains_key(k), "ambiguous binding must not stamp {k}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// ⚠ A binding failure never costs the record.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn no_work_or_a_failed_lookup_leaves_the_record_unbound_but_present() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/books/"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(1342, serde_json::json!({ "numFound": 0, "docs": [] }))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("q", "id_project_gutenberg:84"))
|
||||||
|
.respond_with(ResponseTemplate::new(500))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let (plugin, _dir) = configured_plugin_against(&server);
|
||||||
|
let records = plugin
|
||||||
|
.handle_query(&GatewayQuery::from_free_text("alice"), 50)
|
||||||
|
.await
|
||||||
|
.expect("an Open Library failure must not fail the query");
|
||||||
|
assert_eq!(records.len(), 2);
|
||||||
|
for r in &records {
|
||||||
|
assert!(!r.fields.contains_key("openlibraryid"));
|
||||||
|
assert!(!r.fields.contains_key("anchored"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A book surfaced by every browse must not cost a lookup every browse.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn a_binding_answer_is_looked_up_once_per_book() {
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/books/"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(
|
||||||
|
1342,
|
||||||
|
serde_json::json!({ "numFound": 1, "docs": [
|
||||||
|
{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342"] }
|
||||||
|
]}),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
binding(84, serde_json::json!({ "numFound": 0, "docs": [] }))
|
||||||
|
.expect(1)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let (plugin, _dir) = configured_plugin_against(&server);
|
||||||
|
for _ in 0..2 {
|
||||||
|
let records = plugin
|
||||||
|
.handle_query(&GatewayQuery::from_free_text("alice"), 50)
|
||||||
|
.await
|
||||||
|
.expect("handle_query");
|
||||||
|
assert_eq!(
|
||||||
|
records[0].fields.get("openlibraryid").map(String::as_str),
|
||||||
|
Some("OL66554W")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
server.verify().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The SDK's config page shows the lowest upstream id — this one — so the
|
||||||
|
/// Open Library contact must be settable here.
|
||||||
|
#[test]
|
||||||
|
fn the_open_library_contact_is_configurable_on_this_upstream() {
|
||||||
|
let schema = GutenbergPlugin::new().config_schema();
|
||||||
|
assert!(schema.fields.iter().any(|f| f.key == "contact"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn health_reflects_configure_state() {
|
fn health_reflects_configure_state() {
|
||||||
let plugin = GutenbergPlugin::new();
|
let plugin = GutenbergPlugin::new();
|
||||||
|
|||||||
@@ -1,5 +1,13 @@
|
|||||||
//! `book-feeder` library surface — exposes the Gutenberg plugin so integration
|
//! `book-feeder` library surface — exposes the plugins so integration tests
|
||||||
//! tests (and, later, other binaries) can construct it. The binary
|
//! (and, later, other binaries) can construct them. The binary (`main.rs`) is a
|
||||||
//! (`main.rs`) is a thin wrapper over [`meta_feeder_sdk::serve_feeders`].
|
//! thin wrapper over [`meta_feeder_sdk::serve_feeders`].
|
||||||
|
//!
|
||||||
|
//! Two upstreams:
|
||||||
|
//! - [`gutenberg`] — Project Gutenberg editions, the byte tier (`document` × `book`);
|
||||||
|
//! - [`openlibrary`] — Open Library work cards, the identity tier (`card` × `book`),
|
||||||
|
//! over the client in [`openlibrary_client`], which `gutenberg` also uses to
|
||||||
|
//! bind an edition to its work.
|
||||||
|
|
||||||
pub mod gutenberg;
|
pub mod gutenberg;
|
||||||
|
pub mod openlibrary;
|
||||||
|
pub mod openlibrary_client;
|
||||||
|
|||||||
@@ -1,17 +1,27 @@
|
|||||||
//! `book-feeder` — Project Gutenberg feeder sidecar.
|
//! `book-feeder` — the book tier's feeder sidecar.
|
||||||
//!
|
//!
|
||||||
//! Hosts the [`gutenberg::GutenbergPlugin`] over the feeder HTTP contract via
|
//! Hosts two plugins over the feeder HTTP contract via
|
||||||
//! [`meta_feeder_sdk::serve_feeders`]. The gateway core reaches it server-to-
|
//! [`meta_feeder_sdk::serve_feeders`]:
|
||||||
//! server on the internal docker network.
|
//! - [`GutenbergPlugin`] — Project Gutenberg editions (`document` × `book`);
|
||||||
|
//! - [`OpenLibraryPlugin`] — Open Library work cards (`card` × `book`).
|
||||||
|
//!
|
||||||
|
//! The gateway core reaches it server-to-server on the internal docker network.
|
||||||
//!
|
//!
|
||||||
//! Env:
|
//! Env:
|
||||||
//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`)
|
//! - `META_FEEDER_HTTP_LISTEN` — listen addr (default `0.0.0.0:8080`)
|
||||||
//! - `META_FEEDER_STATE_DIR` — per-plugin cache root (default `/data/meta-feeder`)
|
//! - `META_FEEDER_STATE_DIR` — per-plugin cache root (default `/data/meta-feeder`)
|
||||||
|
//! - `OPENLIBRARY_CONTACT` — e-mail/URL sent in the Open Library User-Agent;
|
||||||
|
//! first-boot seed for the `contact` config field (the saved field wins)
|
||||||
|
//! - `OPENLIBRARY_TOP_N`, `OPENLIBRARY_DISCOVERY_N` — optional card result sizing
|
||||||
//! - `RUST_LOG` — tracing filter (default `info`)
|
//! - `RUST_LOG` — tracing filter (default `info`)
|
||||||
|
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
use book_feeder::gutenberg::GutenbergPlugin;
|
use book_feeder::gutenberg::GutenbergPlugin;
|
||||||
|
use book_feeder::openlibrary::OpenLibraryPlugin;
|
||||||
|
use book_feeder::openlibrary_client;
|
||||||
|
use meta_feeder_sdk::plugin::FeederPlugin;
|
||||||
use meta_feeder_sdk::serve_feeders;
|
use meta_feeder_sdk::serve_feeders;
|
||||||
use tracing_subscriber::EnvFilter;
|
use tracing_subscriber::EnvFilter;
|
||||||
|
|
||||||
@@ -27,5 +37,16 @@ async fn main() -> anyhow::Result<()> {
|
|||||||
let state_dir =
|
let state_dir =
|
||||||
std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string());
|
std::env::var("META_FEEDER_STATE_DIR").unwrap_or_else(|_| "/data/meta-feeder".to_string());
|
||||||
|
|
||||||
serve_feeders(vec![Box::new(GutenbergPlugin::new())], state_dir, listen).await
|
// ONE budget for both plugins: the Gutenberg → Open Library binding and the
|
||||||
|
// card plugin call the same upstream, which rate-limits per client, not per
|
||||||
|
// plugin. Two buckets would each think they had the whole allowance.
|
||||||
|
let contact =
|
||||||
|
openlibrary_client::load_contact(&Path::new(&state_dir).join("gateway").join("openlibrary"));
|
||||||
|
let budget = openlibrary_client::budget(contact.is_some());
|
||||||
|
|
||||||
|
let plugins: Vec<Box<dyn FeederPlugin>> = vec![
|
||||||
|
Box::new(GutenbergPlugin::new().with_openlibrary_budget(budget.clone())),
|
||||||
|
Box::new(OpenLibraryPlugin::new().with_budget(budget)),
|
||||||
|
];
|
||||||
|
serve_feeders(plugins, state_dir, listen).await
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,815 @@
|
|||||||
|
//! The Open Library HTTP client: rate budget, redb read-through cache, and the
|
||||||
|
//! calls the book tier makes.
|
||||||
|
//!
|
||||||
|
//! Shared by both upstreams this binary hosts:
|
||||||
|
//! - the `openlibrary` card plugin — search, trending, "new", work lookups;
|
||||||
|
//! - the `gutenberg` plugin — the `id_project_gutenberg` binding lookup
|
||||||
|
//! (reading-model §2.4), which is what lets an edition join its work.
|
||||||
|
//!
|
||||||
|
//! Plain JSON over `reqwest`, like every feeder in the fleet. Shapes confirmed
|
||||||
|
//! against the live API on 2026-09-11: `search.json` answers `{numFound, docs}`,
|
||||||
|
//! `/trending/daily.json` answers `{works}` with the same doc shape (and honours
|
||||||
|
//! `fields=`), and a work JSON's `description` is either a bare string or a
|
||||||
|
//! `{type: "/type/text", value}` object — both occur in the catalogue.
|
||||||
|
|
||||||
|
use std::path::Path;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use meta_feeder_sdk::budget::{Lease, RateBudget};
|
||||||
|
use meta_feeder_sdk::cache::MidhashCache;
|
||||||
|
use meta_feeder_sdk::config::ConfigField;
|
||||||
|
use meta_feeder_sdk::types::GatewayError;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use serde_json::Value;
|
||||||
|
use tracing::{debug, warn};
|
||||||
|
|
||||||
|
/// Canonical Open Library base URL. Overridable for tests (wiremock).
|
||||||
|
pub const DEFAULT_API_BASE: &str = "https://openlibrary.org";
|
||||||
|
|
||||||
|
/// Cover image base. `-L` is the large rendition; the gateway seeds whatever
|
||||||
|
/// `poster_url` names, so one size is published, not three.
|
||||||
|
const COVERS_BASE: &str = "https://covers.openlibrary.org/b/id";
|
||||||
|
|
||||||
|
/// Config key (and env seed) for the operator contact sent in the User-Agent.
|
||||||
|
pub const CONTACT_KEY: &str = "contact";
|
||||||
|
pub const CONTACT_ENV: &str = "OPENLIBRARY_CONTACT";
|
||||||
|
|
||||||
|
/// How long a caller waits for a rate-budget lease.
|
||||||
|
///
|
||||||
|
/// Three callers, three patiences. A **search** has someone watching a spinner.
|
||||||
|
/// A **browse row** is painted by a timer and is worth less late than never
|
||||||
|
/// blocking the search behind it. A **binding lookup** rides on a Gutenberg
|
||||||
|
/// result that is already in hand, and must never make that result late: an
|
||||||
|
/// unbound edition is still a readable edition, and the next query binds it.
|
||||||
|
pub const SEARCH_DEADLINE: Duration = Duration::from_secs(20);
|
||||||
|
pub const BROWSE_DEADLINE: Duration = Duration::from_secs(5);
|
||||||
|
pub const BINDING_DEADLINE: Duration = Duration::from_secs(2);
|
||||||
|
|
||||||
|
/// HTTP timeout for a single upstream call.
|
||||||
|
const HTTP_TIMEOUT_SECS: u64 = 20;
|
||||||
|
|
||||||
|
/// Attempts per call when the request never produced a response. openlibrary.org
|
||||||
|
/// refused connections outright for a stretch while this tier was being planned;
|
||||||
|
/// one retry covers a blink without hammering an upstream that answered.
|
||||||
|
const SEND_ATTEMPTS: usize = 2;
|
||||||
|
const RETRY_BACKOFF_MS: u64 = 250;
|
||||||
|
|
||||||
|
/// The field selection for every doc-shaped request, written once so a card
|
||||||
|
/// resolved by search and the same card resolved by id are byte-identical.
|
||||||
|
const SEARCH_FIELDS: &str = "key,title,author_name,cover_i,first_publish_year,language";
|
||||||
|
|
||||||
|
/// Cache lifetimes.
|
||||||
|
///
|
||||||
|
/// ⚠ Unlike AniList's, these lists are not immutable. `trending/daily` changes
|
||||||
|
/// every day and `sort=new` every hour; a permanent cache would freeze the
|
||||||
|
/// Popular Books row on whatever it showed the first time. Work JSON and author
|
||||||
|
/// names are edited rarely and are cached long.
|
||||||
|
const DAY: u64 = 86_400;
|
||||||
|
const WORK_TTL_SECS: u64 = 30 * DAY;
|
||||||
|
const SEARCH_TTL_SECS: u64 = DAY;
|
||||||
|
const BROWSE_TTL_SECS: u64 = 6 * 3600;
|
||||||
|
/// A binding is a fact Open Library records; it is re-checked monthly. A miss is
|
||||||
|
/// re-checked weekly, because Open Library librarians add Gutenberg ids to
|
||||||
|
/// editions all the time and a permanent negative would never see them.
|
||||||
|
const BINDING_HIT_TTL_SECS: u64 = 30 * DAY;
|
||||||
|
const BINDING_MISS_TTL_SECS: u64 = 7 * DAY;
|
||||||
|
|
||||||
|
/// Authors resolved on the work-JSON fallback. Each is one more request.
|
||||||
|
const MAX_AUTHORS: usize = 4;
|
||||||
|
|
||||||
|
/// Build the process budget for Open Library.
|
||||||
|
///
|
||||||
|
/// Open Library asks API clients to identify themselves (app name + contact in
|
||||||
|
/// the User-Agent) and throttles anonymous traffic first, so an identified
|
||||||
|
/// client gets a larger bucket. The exact published ceiling could not be read
|
||||||
|
/// back from the developer page when this was written — an overestimate is
|
||||||
|
/// safe regardless: a 429 freezes the whole bucket for its `Retry-After` via
|
||||||
|
/// `note_throttled`, rather than each caller discovering it alone.
|
||||||
|
pub fn budget(identified: bool) -> Arc<RateBudget> {
|
||||||
|
if identified {
|
||||||
|
RateBudget::new(3.0, 6.0)
|
||||||
|
} else {
|
||||||
|
RateBudget::new(1.0, 3.0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The operator contact, file-or-env.
|
||||||
|
///
|
||||||
|
/// ⚠ **Looked up in both plugins' config dirs, not just the caller's.** The
|
||||||
|
/// SDK's `/config` page edits the *lowest* hosted upstream id, which in this
|
||||||
|
/// binary is `gutenberg`, so a contact saved through the dashboard lands in
|
||||||
|
/// `gateway/gutenberg/config.json` — yet both upstreams send requests to Open
|
||||||
|
/// Library and both must identify themselves. File wins over env, as for every
|
||||||
|
/// feeder config.
|
||||||
|
pub fn load_contact(cache_dir: &Path) -> Option<String> {
|
||||||
|
let mut dirs = vec![cache_dir.to_path_buf()];
|
||||||
|
if let Some(parent) = cache_dir.parent() {
|
||||||
|
for id in ["gutenberg", "openlibrary"] {
|
||||||
|
dirs.push(parent.join(id));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for dir in dirs {
|
||||||
|
let Ok(bytes) = std::fs::read(dir.join("config.json")) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let Ok(v) = serde_json::from_slice::<Value>(&bytes) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if let Some(c) = v.get(CONTACT_KEY).and_then(Value::as_str).and_then(sanitize_contact) {
|
||||||
|
return Some(c);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
env_contact()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The first-boot seed: `OPENLIBRARY_CONTACT`.
|
||||||
|
pub fn env_contact() -> Option<String> {
|
||||||
|
std::env::var(CONTACT_ENV).ok().as_deref().and_then(sanitize_contact)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The config field both plugins declare (see [`load_contact`] for why both).
|
||||||
|
pub fn contact_field() -> ConfigField {
|
||||||
|
ConfigField::text(CONTACT_KEY, "Open Library contact").with_help(
|
||||||
|
"An e-mail address or URL sent in the User-Agent of every Open Library request \
|
||||||
|
(the openlibrary cards and the gutenberg → Open Library binding). Open Library \
|
||||||
|
asks API clients to identify themselves; an identified client gets a larger \
|
||||||
|
request budget.",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Reduce a contact to something a header can carry.
|
||||||
|
///
|
||||||
|
/// `build_http_client` treats a client build failure as impossible and panics,
|
||||||
|
/// and an invalid header value is exactly such a failure — so a stray newline
|
||||||
|
/// in a config field must not reach it. Parentheses are dropped too: they would
|
||||||
|
/// close the User-Agent comment early.
|
||||||
|
fn sanitize_contact(raw: &str) -> Option<String> {
|
||||||
|
let c: String = raw
|
||||||
|
.chars()
|
||||||
|
.filter(|c| c.is_ascii_graphic() || *c == ' ')
|
||||||
|
.filter(|c| !matches!(c, '(' | ')'))
|
||||||
|
.take(128)
|
||||||
|
.collect();
|
||||||
|
let c = c.trim();
|
||||||
|
(!c.is_empty()).then(|| c.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn user_agent(contact: Option<&str>) -> String {
|
||||||
|
let base = concat!(
|
||||||
|
"meta-feeder-book/",
|
||||||
|
env!("CARGO_PKG_VERSION"),
|
||||||
|
" (gateway:openlibrary; +https://github.com/worph/meta-feeder-book"
|
||||||
|
);
|
||||||
|
match contact {
|
||||||
|
Some(c) => format!("{base}; {c})"),
|
||||||
|
None => format!("{base})"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Is `s` a bare Open Library **work** id (`OL66554W`)?
|
||||||
|
///
|
||||||
|
/// ⚠ The `W` is the point. `OL…M` is an edition and `OL…A` an author, and the
|
||||||
|
/// search index does return edition keys under `/works/` for some junk records
|
||||||
|
/// (`/works/OL12710531M`, observed live on a `first_publish_year` query) — an
|
||||||
|
/// edition minted as a card would be a second tile for every printing.
|
||||||
|
pub fn is_work_id(s: &str) -> bool {
|
||||||
|
s.len() > 3
|
||||||
|
&& s.starts_with("OL")
|
||||||
|
&& s.ends_with('W')
|
||||||
|
&& s[2..s.len() - 1].bytes().all(|b| b.is_ascii_digit())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One search/trending doc — the subset this feeder reads.
|
||||||
|
///
|
||||||
|
/// ⚠ Nothing mutable is read: no `edition_count`, `ratings_*`,
|
||||||
|
/// `want_to_read_count` or `ebook_access`. All drift without the work changing
|
||||||
|
/// (METADATA_KEYS rule #4).
|
||||||
|
#[derive(Debug, Clone, Default, Deserialize)]
|
||||||
|
pub struct SearchDoc {
|
||||||
|
#[serde(default)]
|
||||||
|
pub key: String,
|
||||||
|
#[serde(default)]
|
||||||
|
pub title: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub author_name: Vec<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub cover_i: Option<i64>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub first_publish_year: Option<i32>,
|
||||||
|
#[serde(default)]
|
||||||
|
pub id_project_gutenberg: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SearchDoc {
|
||||||
|
/// The bare work id, when the key names a work.
|
||||||
|
pub fn work_id(&self) -> Option<&str> {
|
||||||
|
self.key.strip_prefix("/works/").filter(|id| is_work_id(id))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The large cover URL. Open Library uses `-1` for "no cover" in `covers`
|
||||||
|
/// arrays, so anything not positive is absent.
|
||||||
|
pub fn cover_url(&self) -> Option<String> {
|
||||||
|
self.cover_i
|
||||||
|
.filter(|c| *c > 0)
|
||||||
|
.map(|c| format!("{COVERS_BASE}/{c}-L.jpg"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// What a work JSON contributes to a card.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct WorkText {
|
||||||
|
/// The id the work JSON actually lives under — differs from the id asked
|
||||||
|
/// for when that one was merged away (see [`OpenLibraryClient::work_json`]).
|
||||||
|
pub id: String,
|
||||||
|
pub description: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Cache envelope: the body plus when it was fetched, so lists can expire.
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
struct Cached {
|
||||||
|
at: u64,
|
||||||
|
body: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_secs())
|
||||||
|
.unwrap_or(0)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct OpenLibraryClient {
|
||||||
|
http: reqwest::Client,
|
||||||
|
budget: Arc<RateBudget>,
|
||||||
|
cache: Option<MidhashCache>,
|
||||||
|
/// Overridable so tests can point the client at a wiremock server.
|
||||||
|
api_base: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl OpenLibraryClient {
|
||||||
|
pub fn new(
|
||||||
|
budget: Arc<RateBudget>,
|
||||||
|
cache: Option<MidhashCache>,
|
||||||
|
api_base: Option<String>,
|
||||||
|
contact: Option<&str>,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
http: meta_feeder_sdk::common::build_http_client(
|
||||||
|
HTTP_TIMEOUT_SECS,
|
||||||
|
&user_agent(contact),
|
||||||
|
None,
|
||||||
|
),
|
||||||
|
budget,
|
||||||
|
cache,
|
||||||
|
api_base: api_base.unwrap_or_else(|| DEFAULT_API_BASE.to_string()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cache_get(&self, key: &str) -> Option<Cached> {
|
||||||
|
let hit = self.cache.as_ref()?.get_misc(key).ok().flatten()?;
|
||||||
|
serde_json::from_str::<Cached>(&hit).ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cache_put(&self, key: &str, body: &Value) {
|
||||||
|
let Some(c) = &self.cache else { return };
|
||||||
|
let envelope = Cached {
|
||||||
|
at: now_secs(),
|
||||||
|
body: body.clone(),
|
||||||
|
};
|
||||||
|
if let Ok(s) = serde_json::to_string(&envelope) {
|
||||||
|
if let Err(e) = c.put_misc(key, &s) {
|
||||||
|
warn!(upstream = "openlibrary", key, error = ?e, "cache put failed (non-fatal)");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One GET, budget-gated, never cached.
|
||||||
|
async fn get(
|
||||||
|
&self,
|
||||||
|
path: &str,
|
||||||
|
query: &[(&str, String)],
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Value, GatewayError> {
|
||||||
|
if matches!(self.budget.acquire(deadline).await, Lease::DeadlineExceeded) {
|
||||||
|
// Transient, not Permanent: this upstream is busy, not broken.
|
||||||
|
return Err(GatewayError::Transient(
|
||||||
|
"openlibrary: rate budget deadline exceeded".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let url = format!("{}{path}", self.api_base.trim_end_matches('/'));
|
||||||
|
let mut last = String::new();
|
||||||
|
let mut response = None;
|
||||||
|
for attempt in 1..=SEND_ATTEMPTS {
|
||||||
|
match self.http.get(&url).query(query).send().await {
|
||||||
|
Ok(r) => {
|
||||||
|
response = Some(r);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
last = e.to_string();
|
||||||
|
warn!(upstream = "openlibrary", path, attempt, error = %last, "transport failure");
|
||||||
|
if attempt < SEND_ATTEMPTS {
|
||||||
|
tokio::time::sleep(Duration::from_millis(RETRY_BACKOFF_MS)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let resp = response.ok_or_else(|| {
|
||||||
|
GatewayError::Transient(format!("openlibrary: GET {path}: {last}"))
|
||||||
|
})?;
|
||||||
|
// 429 carries Retry-After; freeze the whole bucket, not just this call.
|
||||||
|
if let Err(e) = meta_feeder_sdk::common::map_status(&resp) {
|
||||||
|
if let GatewayError::RateLimited { retry_after_s } = &e {
|
||||||
|
self.budget
|
||||||
|
.note_throttled(Duration::from_secs(*retry_after_s as u64));
|
||||||
|
}
|
||||||
|
return Err(e);
|
||||||
|
}
|
||||||
|
resp.json::<Value>()
|
||||||
|
.await
|
||||||
|
.map_err(|e| GatewayError::Transient(format!("openlibrary: bad json from {path}: {e}")))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// [`Self::get`] behind the read-through cache. Only successes are cached:
|
||||||
|
/// a transport blink or a 5xx must never be remembered as "nothing here".
|
||||||
|
async fn get_cached(
|
||||||
|
&self,
|
||||||
|
key: &str,
|
||||||
|
ttl_secs: u64,
|
||||||
|
path: &str,
|
||||||
|
query: &[(&str, String)],
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Value, GatewayError> {
|
||||||
|
if let Some(c) = self.cache_get(key) {
|
||||||
|
if now_secs().saturating_sub(c.at) <= ttl_secs {
|
||||||
|
return Ok(c.body);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let body = self.get(path, query, deadline).await?;
|
||||||
|
self.cache_put(key, &body);
|
||||||
|
Ok(body)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Free-text search.
|
||||||
|
pub async fn search(
|
||||||
|
&self,
|
||||||
|
text: &str,
|
||||||
|
limit: usize,
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Vec<SearchDoc>, GatewayError> {
|
||||||
|
let normalised = text.split_whitespace().collect::<Vec<_>>().join(" ").to_lowercase();
|
||||||
|
let body = self
|
||||||
|
.get_cached(
|
||||||
|
&format!("openlibrary:search:{limit}:{normalised}"),
|
||||||
|
SEARCH_TTL_SECS,
|
||||||
|
"/search.json",
|
||||||
|
&[
|
||||||
|
("q", text.trim().to_string()),
|
||||||
|
("fields", SEARCH_FIELDS.into()),
|
||||||
|
("limit", limit.to_string()),
|
||||||
|
],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(parse_docs(body.get("docs")))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The `popular:true` row: what Open Library readers opened today.
|
||||||
|
///
|
||||||
|
/// The ordering is a popularity signal, but no count is read or stored.
|
||||||
|
pub async fn trending(
|
||||||
|
&self,
|
||||||
|
limit: usize,
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Vec<SearchDoc>, GatewayError> {
|
||||||
|
let body = self
|
||||||
|
.get_cached(
|
||||||
|
&format!("openlibrary:trending:daily:{limit}"),
|
||||||
|
BROWSE_TTL_SECS,
|
||||||
|
"/trending/daily.json",
|
||||||
|
&[("fields", SEARCH_FIELDS.into()), ("limit", limit.to_string())],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(parse_docs(body.get("works")))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The `fresh:true` row: recently added works first published this year or
|
||||||
|
/// last.
|
||||||
|
///
|
||||||
|
/// ⚠ `search.json` rejects `q=*` (422, "Query too short"), and `sort=new`
|
||||||
|
/// over a broad query is dominated by coverless catalogue stubs. The year
|
||||||
|
/// window keeps the row about new books; the card gate (cover + description)
|
||||||
|
/// drops the stubs that remain, so the caller over-fetches.
|
||||||
|
pub async fn fresh(
|
||||||
|
&self,
|
||||||
|
limit: usize,
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Vec<SearchDoc>, GatewayError> {
|
||||||
|
let year = 1970 + now_secs() / 31_556_952;
|
||||||
|
let body = self
|
||||||
|
.get_cached(
|
||||||
|
&format!("openlibrary:fresh:{year}:{limit}"),
|
||||||
|
BROWSE_TTL_SECS,
|
||||||
|
"/search.json",
|
||||||
|
&[
|
||||||
|
("q", format!("first_publish_year:[{} TO {year}]", year - 1)),
|
||||||
|
("sort", "new".into()),
|
||||||
|
("fields", SEARCH_FIELDS.into()),
|
||||||
|
("limit", limit.to_string()),
|
||||||
|
],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(parse_docs(body.get("docs")))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Resolve one work id to a doc — the anchored lookup.
|
||||||
|
///
|
||||||
|
/// Search by key first, so the record matches what a search would have
|
||||||
|
/// produced (same fields, same `first_publish_year`; the work JSON's
|
||||||
|
/// `first_publish_date` is frequently an edition's date — `1853` for a novel
|
||||||
|
/// first published in 1813). The work JSON plus its author records are the
|
||||||
|
/// fallback for a work the search index does not hold; that path carries no
|
||||||
|
/// year, because absent is better than wrong.
|
||||||
|
pub async fn doc_by_work_id(
|
||||||
|
&self,
|
||||||
|
id: &str,
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Option<SearchDoc>, GatewayError> {
|
||||||
|
let body = self
|
||||||
|
.get_cached(
|
||||||
|
&format!("openlibrary:key:{id}"),
|
||||||
|
WORK_TTL_SECS,
|
||||||
|
"/search.json",
|
||||||
|
&[
|
||||||
|
("q", format!("key:/works/{id}")),
|
||||||
|
("fields", SEARCH_FIELDS.into()),
|
||||||
|
("limit", "1".into()),
|
||||||
|
],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
if let Some(doc) = parse_docs(body.get("docs"))
|
||||||
|
.into_iter()
|
||||||
|
.find(|d| d.work_id() == Some(id))
|
||||||
|
{
|
||||||
|
return Ok(Some(doc));
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some((work_id, work)) = self.work_json(id, deadline).await? else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let author_keys: Vec<String> = work
|
||||||
|
.get("authors")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.map(|a| {
|
||||||
|
a.iter()
|
||||||
|
.filter_map(|x| x.pointer("/author/key").and_then(Value::as_str))
|
||||||
|
.filter_map(|k| k.strip_prefix("/authors/"))
|
||||||
|
.map(str::to_string)
|
||||||
|
.take(MAX_AUTHORS)
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default();
|
||||||
|
let mut author_name = Vec::new();
|
||||||
|
for author in author_keys {
|
||||||
|
match self.author_name(&author, deadline).await {
|
||||||
|
Ok(Some(name)) => author_name.push(name),
|
||||||
|
Ok(None) => {}
|
||||||
|
Err(e) => {
|
||||||
|
debug!(upstream = "openlibrary", author, error = ?e, "author lookup failed")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Some(SearchDoc {
|
||||||
|
key: format!("/works/{work_id}"),
|
||||||
|
title: work.get("title").and_then(Value::as_str).map(str::to_string),
|
||||||
|
author_name,
|
||||||
|
cover_i: work
|
||||||
|
.get("covers")
|
||||||
|
.and_then(Value::as_array)
|
||||||
|
.and_then(|a| a.iter().filter_map(Value::as_i64).find(|c| *c > 0)),
|
||||||
|
first_publish_year: None,
|
||||||
|
id_project_gutenberg: Vec::new(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn author_name(&self, id: &str, deadline: Duration) -> Result<Option<String>, GatewayError> {
|
||||||
|
let body = match self
|
||||||
|
.get_cached(
|
||||||
|
&format!("openlibrary:author:{id}"),
|
||||||
|
WORK_TTL_SECS,
|
||||||
|
&format!("/authors/{id}.json"),
|
||||||
|
&[],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(b) => b,
|
||||||
|
Err(GatewayError::NotFound) => return Ok(None),
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
};
|
||||||
|
Ok(body
|
||||||
|
.get("name")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.map(str::trim)
|
||||||
|
.filter(|s| !s.is_empty())
|
||||||
|
.map(str::to_string))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The work JSON and the id it actually lives under, following one merge
|
||||||
|
/// redirect.
|
||||||
|
///
|
||||||
|
/// ⚠ A merged work answers **200** with `{"type": {"key": "/type/redirect"},
|
||||||
|
/// "location": "/works/OL…W"}` (observed live: `OL45883W` → `OL45804W`) —
|
||||||
|
/// not an HTTP redirect, so an HTTP client follows nothing and a naive
|
||||||
|
/// reader finds no description.
|
||||||
|
async fn work_json(
|
||||||
|
&self,
|
||||||
|
id: &str,
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Option<(String, Value)>, GatewayError> {
|
||||||
|
let mut id = id.to_string();
|
||||||
|
for _ in 0..2 {
|
||||||
|
let body = match self
|
||||||
|
.get_cached(
|
||||||
|
&format!("openlibrary:work:{id}"),
|
||||||
|
WORK_TTL_SECS,
|
||||||
|
&format!("/works/{id}.json"),
|
||||||
|
&[],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(b) => b,
|
||||||
|
Err(GatewayError::NotFound) => return Ok(None),
|
||||||
|
Err(e) => return Err(e),
|
||||||
|
};
|
||||||
|
if body.pointer("/type/key").and_then(Value::as_str) != Some("/type/redirect") {
|
||||||
|
return Ok(Some((id, body)));
|
||||||
|
}
|
||||||
|
match body
|
||||||
|
.get("location")
|
||||||
|
.and_then(Value::as_str)
|
||||||
|
.and_then(|l| l.strip_prefix("/works/"))
|
||||||
|
.filter(|l| is_work_id(l))
|
||||||
|
{
|
||||||
|
Some(next) => id = next.to_string(),
|
||||||
|
None => return Ok(None),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A work's canonical id and plain-text description.
|
||||||
|
pub async fn work_text(&self, id: &str, deadline: Duration) -> Result<Option<WorkText>, GatewayError> {
|
||||||
|
Ok(self.work_json(id, deadline).await?.map(|(id, work)| WorkText {
|
||||||
|
id,
|
||||||
|
description: text_value(work.get("description")).and_then(|d| clean_description(&d)),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The Open Library work that records Project Gutenberg etext `gutenberg_id`.
|
||||||
|
///
|
||||||
|
/// `Some` only when **exactly one** work lists it — an identifier link Open
|
||||||
|
/// Library itself records, which is what grades a binding Anchored
|
||||||
|
/// (reading-model §2.4). Zero works is no binding; two or more is an
|
||||||
|
/// unmerged duplicate in Open Library, and picking one would be a guess.
|
||||||
|
///
|
||||||
|
/// ⚠ Never a title or author search. That is a Matched-grade binding and is
|
||||||
|
/// out of scope here; emitting it under an anchored claim would promote a
|
||||||
|
/// resemblance to a proof.
|
||||||
|
pub async fn gutenberg_work(
|
||||||
|
&self,
|
||||||
|
gutenberg_id: u64,
|
||||||
|
deadline: Duration,
|
||||||
|
) -> Result<Option<String>, GatewayError> {
|
||||||
|
let key = format!("openlibrary:gutenberg:{gutenberg_id}");
|
||||||
|
if let Some(c) = self.cache_get(&key) {
|
||||||
|
let work = c.body.get("work").and_then(Value::as_str).map(str::to_string);
|
||||||
|
let ttl = if work.is_some() {
|
||||||
|
BINDING_HIT_TTL_SECS
|
||||||
|
} else {
|
||||||
|
BINDING_MISS_TTL_SECS
|
||||||
|
};
|
||||||
|
if now_secs().saturating_sub(c.at) <= ttl {
|
||||||
|
return Ok(work);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let body = self
|
||||||
|
.get(
|
||||||
|
"/search.json",
|
||||||
|
&[
|
||||||
|
("q", format!("id_project_gutenberg:{gutenberg_id}")),
|
||||||
|
("fields", "key,id_project_gutenberg".into()),
|
||||||
|
// Two, not one: `limit=1` would make "exactly one" unprovable.
|
||||||
|
("limit", "2".into()),
|
||||||
|
],
|
||||||
|
deadline,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let work = single_gutenberg_work(&body, gutenberg_id);
|
||||||
|
self.cache_put(&key, &serde_json::json!({ "work": work }));
|
||||||
|
Ok(work)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Decide a binding from a `q=id_project_gutenberg:<id>` response.
|
||||||
|
fn single_gutenberg_work(body: &Value, gutenberg_id: u64) -> Option<String> {
|
||||||
|
let docs = parse_docs(body.get("docs"));
|
||||||
|
let total = body
|
||||||
|
.get("numFound")
|
||||||
|
.or_else(|| body.get("num_found"))
|
||||||
|
.and_then(Value::as_u64)
|
||||||
|
.unwrap_or(docs.len() as u64);
|
||||||
|
if total != 1 || docs.len() != 1 {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let doc = &docs[0];
|
||||||
|
// The doc must itself list the etext. The query is an exact field match, so
|
||||||
|
// this only fails on an index quirk — and a binding graded Anchored cannot
|
||||||
|
// rest on a quirk.
|
||||||
|
let wanted = gutenberg_id.to_string();
|
||||||
|
if !doc.id_project_gutenberg.iter().any(|g| g.trim() == wanted) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
doc.work_id().map(str::to_string)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_docs(v: Option<&Value>) -> Vec<SearchDoc> {
|
||||||
|
v.and_then(Value::as_array)
|
||||||
|
.map(|a| {
|
||||||
|
a.iter()
|
||||||
|
.filter_map(|d| serde_json::from_value::<SearchDoc>(d.clone()).ok())
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A text field that is either `"…"` or `{"type": "/type/text", "value": "…"}`.
|
||||||
|
fn text_value(v: Option<&Value>) -> Option<String> {
|
||||||
|
match v? {
|
||||||
|
Value::String(s) => Some(s.clone()),
|
||||||
|
Value::Object(o) => o.get("value").and_then(Value::as_str).map(str::to_string),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Community descriptions carry Windows line endings and, often, a trailing
|
||||||
|
/// `----------` rule followed by "Also contained in:" link lists. The rule and
|
||||||
|
/// what follows are catalogue furniture, not synopsis.
|
||||||
|
fn clean_description(raw: &str) -> Option<String> {
|
||||||
|
let text = raw.replace("\r\n", "\n").replace('\r', "\n");
|
||||||
|
let body = text.split("\n----").next().unwrap_or_default().trim();
|
||||||
|
(!body.is_empty()).then(|| body.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn work_ids_are_works_only() {
|
||||||
|
assert!(is_work_id("OL66554W"));
|
||||||
|
assert!(!is_work_id("OL12710531M"), "an edition key is not a work");
|
||||||
|
assert!(!is_work_id("OL21594A"), "an author key is not a work");
|
||||||
|
assert!(!is_work_id("OLW"));
|
||||||
|
assert!(!is_work_id("/works/OL66554W"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn description_reads_both_live_shapes() {
|
||||||
|
let plain = serde_json::json!({ "description": "A novel." });
|
||||||
|
let typed = serde_json::json!({ "description": { "type": "/type/text", "value": "A novel." } });
|
||||||
|
assert_eq!(text_value(plain.get("description")).as_deref(), Some("A novel."));
|
||||||
|
assert_eq!(text_value(typed.get("description")).as_deref(), Some("A novel."));
|
||||||
|
assert_eq!(text_value(serde_json::json!({ "description": 3 }).get("description")), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn description_drops_the_link_list_after_the_rule() {
|
||||||
|
let raw = "First para.\r\n\r\nSecond para.\r\n\r\n----------\r\nAlso contained in:\r\n[x](/works/OL1W)";
|
||||||
|
assert_eq!(
|
||||||
|
clean_description(raw).as_deref(),
|
||||||
|
Some("First para.\n\nSecond para.")
|
||||||
|
);
|
||||||
|
assert_eq!(clean_description(" \r\n "), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cover_minus_one_is_no_cover() {
|
||||||
|
let mut d = SearchDoc {
|
||||||
|
key: "/works/OL66554W".into(),
|
||||||
|
cover_i: Some(-1),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(d.cover_url(), None);
|
||||||
|
d.cover_i = Some(14348537);
|
||||||
|
assert_eq!(
|
||||||
|
d.cover_url().as_deref(),
|
||||||
|
Some("https://covers.openlibrary.org/b/id/14348537-L.jpg")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_binding_needs_exactly_one_work_that_lists_the_etext() {
|
||||||
|
let one = serde_json::json!({ "numFound": 1, "docs": [
|
||||||
|
{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342", "42671"] }
|
||||||
|
]});
|
||||||
|
assert_eq!(single_gutenberg_work(&one, 1342).as_deref(), Some("OL66554W"));
|
||||||
|
|
||||||
|
let none = serde_json::json!({ "numFound": 0, "docs": [] });
|
||||||
|
assert_eq!(single_gutenberg_work(&none, 84), None);
|
||||||
|
|
||||||
|
// `limit=2` returned two docs: an unmerged duplicate, not a binding.
|
||||||
|
let two = serde_json::json!({ "numFound": 2, "docs": [
|
||||||
|
{ "key": "/works/OL1W", "id_project_gutenberg": ["11"] },
|
||||||
|
{ "key": "/works/OL2W", "id_project_gutenberg": ["11"] }
|
||||||
|
]});
|
||||||
|
assert_eq!(single_gutenberg_work(&two, 11), None);
|
||||||
|
|
||||||
|
// More found than returned also counts as ambiguous.
|
||||||
|
let more = serde_json::json!({ "numFound": 3, "docs": [
|
||||||
|
{ "key": "/works/OL1W", "id_project_gutenberg": ["11"] }
|
||||||
|
]});
|
||||||
|
assert_eq!(single_gutenberg_work(&more, 11), None);
|
||||||
|
|
||||||
|
// A doc that does not list the etext itself proves nothing.
|
||||||
|
let unlisted = serde_json::json!({ "numFound": 1, "docs": [
|
||||||
|
{ "key": "/works/OL1W", "id_project_gutenberg": ["999"] }
|
||||||
|
]});
|
||||||
|
assert_eq!(single_gutenberg_work(&unlisted, 11), None);
|
||||||
|
|
||||||
|
// An edition key is never bound as a work.
|
||||||
|
let edition = serde_json::json!({ "numFound": 1, "docs": [
|
||||||
|
{ "key": "/works/OL12710531M", "id_project_gutenberg": ["11"] }
|
||||||
|
]});
|
||||||
|
assert_eq!(single_gutenberg_work(&edition, 11), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn contact_is_made_header_safe() {
|
||||||
|
assert_eq!(
|
||||||
|
sanitize_contact(" [email protected]\r\n(x) ").as_deref(),
|
||||||
|
Some("[email protected]")
|
||||||
|
);
|
||||||
|
assert_eq!(sanitize_contact("\n"), None);
|
||||||
|
assert!(user_agent(Some("[email protected]")).ends_with("; [email protected])"));
|
||||||
|
// Must build without panicking.
|
||||||
|
let _ = OpenLibraryClient::new(budget(false), None, None, Some("[email protected]"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_dashboard_saved_contact_is_found_from_either_plugin_dir() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let gutenberg = root.path().join("gutenberg");
|
||||||
|
let openlibrary = root.path().join("openlibrary");
|
||||||
|
std::fs::create_dir_all(&gutenberg).unwrap();
|
||||||
|
std::fs::create_dir_all(&openlibrary).unwrap();
|
||||||
|
// The SDK's config page saves under the lowest upstream id.
|
||||||
|
std::fs::write(
|
||||||
|
gutenberg.join("config.json"),
|
||||||
|
r#"{"contact":"[email protected]"}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(load_contact(&openlibrary).as_deref(), Some("[email protected]"));
|
||||||
|
assert_eq!(load_contact(&gutenberg).as_deref(), Some("[email protected]"));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A cached miss is not re-asked inside its window, and a failure is never
|
||||||
|
/// cached at all.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn binding_lookups_cache_answers_but_never_failures() {
|
||||||
|
use wiremock::matchers::{method, path, query_param};
|
||||||
|
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||||
|
|
||||||
|
let server = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("q", "id_project_gutenberg:84"))
|
||||||
|
.respond_with(
|
||||||
|
ResponseTemplate::new(200).set_body_json(serde_json::json!({ "numFound": 0, "docs": [] })),
|
||||||
|
)
|
||||||
|
.expect(1)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("q", "id_project_gutenberg:11"))
|
||||||
|
.respond_with(ResponseTemplate::new(503))
|
||||||
|
.expect(2)
|
||||||
|
.mount(&server)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let cache = MidhashCache::open(dir.path()).unwrap();
|
||||||
|
let c = OpenLibraryClient::new(RateBudget::new(100.0, 100.0), Some(cache), Some(server.uri()), None);
|
||||||
|
|
||||||
|
assert_eq!(c.gutenberg_work(84, SEARCH_DEADLINE).await.unwrap(), None);
|
||||||
|
assert_eq!(c.gutenberg_work(84, SEARCH_DEADLINE).await.unwrap(), None);
|
||||||
|
assert!(c.gutenberg_work(11, SEARCH_DEADLINE).await.is_err());
|
||||||
|
assert!(c.gutenberg_work(11, SEARCH_DEADLINE).await.is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,24 +1,36 @@
|
|||||||
//! End-to-end contract test for the feeder HTTP surface: boots the real
|
//! End-to-end contract test for the feeder HTTP surface: boots the real
|
||||||
//! `serve_feeders` router (book-feeder hosting GutenbergPlugin) on an ephemeral
|
//! `serve_feeders` router (book-feeder hosting GutenbergPlugin AND
|
||||||
//! port, pointed at a wiremock Gutendex, and drives `/manifest`, `/query`, and
|
//! OpenLibraryPlugin, as `main.rs` does) on an ephemeral port, pointed at one
|
||||||
//! `/compute` over HTTP. Deterministic — no live Gutendex.
|
//! wiremock server that plays Gutendex, the Gutenberg RDF host and Open Library,
|
||||||
|
//! and drives `/manifest`, `/query`, and `/compute` over HTTP. Deterministic —
|
||||||
|
//! no live upstream.
|
||||||
|
|
||||||
use book_feeder::gutenberg::GutenbergPlugin;
|
use book_feeder::gutenberg::GutenbergPlugin;
|
||||||
|
use book_feeder::openlibrary::OpenLibraryPlugin;
|
||||||
|
use book_feeder::openlibrary_client;
|
||||||
use meta_feeder_sdk::{
|
use meta_feeder_sdk::{
|
||||||
configure_plugins, hash, router, ComputeRequest, ComputeResponse, GatewayQuery, HashKindDto,
|
configure_plugins, hash, router, ComputeRequest, ComputeResponse, FeederPlugin, GatewayQuery,
|
||||||
ManifestResponse, QueryRequest, QueryResponse,
|
HashKindDto, ManifestResponse, QueryRequest, QueryResponse,
|
||||||
};
|
};
|
||||||
use wiremock::matchers::{method, path};
|
use wiremock::matchers::{method, path, query_param};
|
||||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||||
|
|
||||||
/// Spawn the feeder router (Gutenberg pointed at `upstream`) on an ephemeral
|
/// Spawn the feeder router (every upstream pointed at `upstream`) on an
|
||||||
/// port; return its base URL. The TempDir is returned so the per-plugin cache
|
/// ephemeral port; return its base URL. The TempDir is returned so the
|
||||||
/// outlives the test.
|
/// per-plugin cache outlives the test.
|
||||||
async fn spawn_feeder(upstream: &MockServer) -> (String, tempfile::TempDir) {
|
async fn spawn_feeder(upstream: &MockServer) -> (String, tempfile::TempDir) {
|
||||||
let dir = tempfile::tempdir().expect("tempdir");
|
let dir = tempfile::tempdir().expect("tempdir");
|
||||||
let plugin = GutenbergPlugin::with_base_url(upstream.uri());
|
let budget = openlibrary_client::budget(false);
|
||||||
let plugins =
|
let plugins: Vec<Box<dyn FeederPlugin>> = vec![
|
||||||
configure_plugins(vec![Box::new(plugin)], dir.path()).expect("configure plugins");
|
Box::new(
|
||||||
|
GutenbergPlugin::with_base_url(upstream.uri())
|
||||||
|
.with_rdf_base_url(upstream.uri())
|
||||||
|
.with_openlibrary_base_url(upstream.uri())
|
||||||
|
.with_openlibrary_budget(budget.clone()),
|
||||||
|
),
|
||||||
|
Box::new(OpenLibraryPlugin::with_api_base(upstream.uri()).with_budget(budget)),
|
||||||
|
];
|
||||||
|
let plugins = configure_plugins(plugins, dir.path()).expect("configure plugins");
|
||||||
let app = router(plugins, "book-feeder-test".to_string(), dir.path());
|
let app = router(plugins, "book-feeder-test".to_string(), dir.path());
|
||||||
|
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||||
@@ -44,7 +56,7 @@ fn books_json() -> serde_json::Value {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn manifest_lists_book_capabilities() {
|
async fn manifest_lists_both_upstreams() {
|
||||||
let upstream = MockServer::start().await;
|
let upstream = MockServer::start().await;
|
||||||
let (base, _dir) = spawn_feeder(&upstream).await;
|
let (base, _dir) = spawn_feeder(&upstream).await;
|
||||||
|
|
||||||
@@ -57,21 +69,36 @@ async fn manifest_lists_book_capabilities() {
|
|||||||
.await
|
.await
|
||||||
.expect("decode manifest");
|
.expect("decode manifest");
|
||||||
|
|
||||||
assert_eq!(manifest.plugins.len(), 1);
|
assert_eq!(manifest.plugins.len(), 2);
|
||||||
let p = &manifest.plugins[0];
|
let gutenberg = manifest.plugins.iter().find(|p| p.id == "gutenberg").expect("gutenberg");
|
||||||
assert_eq!(p.id, "gutenberg");
|
assert_eq!(gutenberg.served_file_types, vec!["document".to_string()]);
|
||||||
assert_eq!(p.served_file_types, vec!["document".to_string()]);
|
assert_eq!(gutenberg.served_content_kinds, vec!["book".to_string()]);
|
||||||
assert_eq!(p.served_content_kinds, vec!["book".to_string()]);
|
let openlibrary = manifest
|
||||||
|
.plugins
|
||||||
|
.iter()
|
||||||
|
.find(|p| p.id == "openlibrary")
|
||||||
|
.expect("openlibrary");
|
||||||
|
assert_eq!(openlibrary.served_file_types, vec!["card".to_string()]);
|
||||||
|
assert_eq!(openlibrary.served_content_kinds, vec!["book".to_string()]);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn query_returns_gutenberg_records() {
|
async fn query_returns_gutenberg_records_with_their_open_library_claim() {
|
||||||
let upstream = MockServer::start().await;
|
let upstream = MockServer::start().await;
|
||||||
Mock::given(method("GET"))
|
Mock::given(method("GET"))
|
||||||
.and(path("/books/"))
|
.and(path("/books/"))
|
||||||
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
.respond_with(ResponseTemplate::new(200).set_body_json(books_json()))
|
||||||
.mount(&upstream)
|
.mount(&upstream)
|
||||||
.await;
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("q", "id_project_gutenberg:1342"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||||
|
"numFound": 1,
|
||||||
|
"docs": [{ "key": "/works/OL66554W", "id_project_gutenberg": ["1342", "42671"] }]
|
||||||
|
})))
|
||||||
|
.mount(&upstream)
|
||||||
|
.await;
|
||||||
let (base, _dir) = spawn_feeder(&upstream).await;
|
let (base, _dir) = spawn_feeder(&upstream).await;
|
||||||
|
|
||||||
let resp: QueryResponse = reqwest::Client::new()
|
let resp: QueryResponse = reqwest::Client::new()
|
||||||
@@ -90,9 +117,75 @@ async fn query_returns_gutenberg_records() {
|
|||||||
|
|
||||||
assert_eq!(resp.records.len(), 1);
|
assert_eq!(resp.records.len(), 1);
|
||||||
let r = &resp.records[0];
|
let r = &resp.records[0];
|
||||||
|
let f = |k: &str| r.fields.get(k).map(String::as_str);
|
||||||
assert_eq!(r.upstream_id, "gutenberg");
|
assert_eq!(r.upstream_id, "gutenberg");
|
||||||
assert_eq!(r.fields.get("gutenbergid").map(String::as_str), Some("1342"));
|
assert_eq!(f("gutenbergid"), Some("1342"));
|
||||||
assert_eq!(r.fields.get("contentKind").map(String::as_str), Some("book"));
|
assert_eq!(f("contentKind"), Some("book"));
|
||||||
|
assert_eq!(f("authors/Jane Austen"), Some("true"));
|
||||||
|
assert_eq!(f("languages/eng"), Some("true"));
|
||||||
|
// The transient claim the gateway transcribes into
|
||||||
|
// `anchoredBy/openlibrary:OL66554W`.
|
||||||
|
assert_eq!(f("openlibraryid"), Some("OL66554W"));
|
||||||
|
assert_eq!(f("anchored"), Some("true"));
|
||||||
|
assert_eq!(f("anchorSource"), Some("openlibrary"));
|
||||||
|
assert_eq!(f("anchorMethod"), None, "an identifier link is not a title match");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn query_returns_an_open_library_card() {
|
||||||
|
let upstream = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("q", "pride"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||||
|
"numFound": 1,
|
||||||
|
"docs": [{
|
||||||
|
"key": "/works/OL66554W",
|
||||||
|
"title": "Pride and Prejudice",
|
||||||
|
"author_name": ["Jane Austen"],
|
||||||
|
"cover_i": 14348537,
|
||||||
|
"first_publish_year": 1813
|
||||||
|
}]
|
||||||
|
})))
|
||||||
|
.mount(&upstream)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/works/OL66554W.json"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||||
|
"key": "/works/OL66554W",
|
||||||
|
"description": "A novel of manners."
|
||||||
|
})))
|
||||||
|
.mount(&upstream)
|
||||||
|
.await;
|
||||||
|
let (base, _dir) = spawn_feeder(&upstream).await;
|
||||||
|
|
||||||
|
let mut query = GatewayQuery::from_free_text("pride");
|
||||||
|
query.filters.insert("fileType".into(), vec!["card".into()]);
|
||||||
|
let resp: QueryResponse = reqwest::Client::new()
|
||||||
|
.post(format!("{base}/query"))
|
||||||
|
.json(&QueryRequest {
|
||||||
|
upstream_id: "openlibrary".to_string(),
|
||||||
|
query,
|
||||||
|
max_results: 10,
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect("POST /query")
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.expect("decode query response");
|
||||||
|
|
||||||
|
assert_eq!(resp.records.len(), 1);
|
||||||
|
let r = &resp.records[0];
|
||||||
|
assert_eq!(r.upstream_id, "openlibrary");
|
||||||
|
assert_eq!(r.record_id, "OL66554W");
|
||||||
|
assert_eq!(r.fields.get("fileType").map(String::as_str), Some("card"));
|
||||||
|
assert_eq!(
|
||||||
|
r.fields
|
||||||
|
.get("cids/bagdsaaaubnxxazlonruwe4tboj4u6tbwgy2tkncx")
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("true")
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -151,6 +244,57 @@ async fn compute_returns_outcome_with_inline_epub_bytes() {
|
|||||||
assert_eq!(hash::compute_ipfs_cid(&decoded), expected_cid);
|
assert_eq!(hash::compute_ipfs_cid(&decoded), expected_cid);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn compute_for_an_open_library_card_is_a_byteless_card_locator() {
|
||||||
|
let upstream = MockServer::start().await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/search.json"))
|
||||||
|
.and(query_param("q", "key:/works/OL45804W"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||||
|
"numFound": 1,
|
||||||
|
"docs": [{
|
||||||
|
"key": "/works/OL45804W",
|
||||||
|
"title": "Fantastic Mr Fox",
|
||||||
|
"author_name": ["Roald Dahl"],
|
||||||
|
"cover_i": 6498519,
|
||||||
|
"first_publish_year": 1970
|
||||||
|
}]
|
||||||
|
})))
|
||||||
|
.mount(&upstream)
|
||||||
|
.await;
|
||||||
|
Mock::given(method("GET"))
|
||||||
|
.and(path("/works/OL45804W.json"))
|
||||||
|
.respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
|
||||||
|
"key": "/works/OL45804W",
|
||||||
|
"description": "A clever fox outwits three farmers."
|
||||||
|
})))
|
||||||
|
.mount(&upstream)
|
||||||
|
.await;
|
||||||
|
let (base, _dir) = spawn_feeder(&upstream).await;
|
||||||
|
|
||||||
|
let resp: ComputeResponse = reqwest::Client::new()
|
||||||
|
.post(format!("{base}/compute"))
|
||||||
|
.json(&ComputeRequest {
|
||||||
|
upstream_id: "openlibrary".to_string(),
|
||||||
|
record_id: "OL45804W".to_string(),
|
||||||
|
})
|
||||||
|
.send()
|
||||||
|
.await
|
||||||
|
.expect("POST /compute")
|
||||||
|
.json()
|
||||||
|
.await
|
||||||
|
.expect("decode compute response");
|
||||||
|
|
||||||
|
assert_eq!(resp.outcomes.len(), 1);
|
||||||
|
let o = &resp.outcomes[0];
|
||||||
|
assert_eq!(o.hash, "bagdsaaaubnxxazlonruwe4tboj4u6tbugu4dancx");
|
||||||
|
assert_eq!(o.hash_kind, HashKindDto::CardLocator);
|
||||||
|
assert!(o.bytes_b64.is_none());
|
||||||
|
assert!(o.file_extension.is_none());
|
||||||
|
let record = o.record.as_ref().expect("the card record rides the outcome");
|
||||||
|
assert_eq!(record.fields.get("title").map(String::as_str), Some("Fantastic Mr Fox"));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn query_unknown_upstream_is_404() {
|
async fn query_unknown_upstream_is_404() {
|
||||||
let upstream = MockServer::start().await;
|
let upstream = MockServer::start().await;
|
||||||
|
|||||||
Reference in New Issue
Block a user