add cache
This commit is contained in:
+179
-68
@@ -451,15 +451,25 @@ export async function process(
|
|||||||
console.log(`[tmdb] Force recompute enabled for ${request.filePath ?? existingMeta?.fileName ?? cid}`);
|
console.log(`[tmdb] Force recompute enabled for ${request.filePath ?? existingMeta?.fileName ?? cid}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get midhash for caching
|
const label = request.filePath ?? existingMeta?.fileName ?? cid;
|
||||||
const midhash = existingMeta?.['cid_midhash256'];
|
|
||||||
|
|
||||||
// Check cache (skip if forceRecompute)
|
// Exact-file cache, keyed by the file's byte-hash. Only records carrying
|
||||||
|
// real file bytes have `cid_midhash256`; gateway/usenet records don't —
|
||||||
|
// which is exactly why the per-tmdbid cache below exists. Stores a full
|
||||||
|
// entry now, but tolerates the legacy raw-payload format from older builds.
|
||||||
|
const midhash = existingMeta?.['cid_midhash256'];
|
||||||
if (midhash && !forceRecompute) {
|
if (midhash && !forceRecompute) {
|
||||||
const cachedData = await readJson<any>(`${midhash}_tmdb.json`);
|
const cached = await readJson<any>(`${midhash}_tmdb.json`);
|
||||||
if (cachedData) {
|
if (cached) {
|
||||||
console.log(`[tmdb] Using cached TMDB data for ${request.filePath ?? existingMeta?.fileName ?? cid}`);
|
console.log(`[tmdb] Using byte-hash cached TMDB data for ${label}`);
|
||||||
await applyTmdbData(metaCore, cid, cachedData);
|
const isEntry = cached.data !== undefined;
|
||||||
|
await applyTmdbData(
|
||||||
|
metaCore,
|
||||||
|
cid,
|
||||||
|
isEntry ? cached.data : cached,
|
||||||
|
// Legacy raw payloads carry no image CIDs → let applyTmdbData recompute.
|
||||||
|
isEntry ? { posterCid: cached.posterCid, backdropCid: cached.backdropCid } : undefined
|
||||||
|
);
|
||||||
await sendCallback({
|
await sendCallback({
|
||||||
taskId: request.taskId,
|
taskId: request.taskId,
|
||||||
status: 'completed',
|
status: 'completed',
|
||||||
@@ -469,54 +479,69 @@ export async function process(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let tmdbData = null;
|
// Resolve the show/movie entry, deduped per tmdbid. A record that already
|
||||||
|
// carries a tmdbid anchor (every gateway torrent/usenet hit does) takes
|
||||||
// Resolve by a known TMDB id first (authoritative — e.g. a gateway
|
// the cached + single-flighted fast path, so N releases of one show share
|
||||||
// anchor or jellyfin-nfo). Completes the full cycle (poster + plot +
|
// ONE TMDB fetch + poster seed instead of N.
|
||||||
// genres) from an id alone, with no fuzzy re-search.
|
let entry: TmdbCacheEntry | null = null;
|
||||||
const knownTmdbId = existingMeta?.tmdbid;
|
const knownTmdbId = existingMeta?.tmdbid;
|
||||||
if (knownTmdbId) {
|
if (knownTmdbId) {
|
||||||
const mediaType =
|
const mediaType =
|
||||||
existingMeta?.videoType === 'tvshow' || existingMeta?.videoType === 'tv'
|
existingMeta?.videoType === 'tvshow' || existingMeta?.videoType === 'tv'
|
||||||
? 'tv'
|
? 'tv'
|
||||||
: 'movie';
|
: 'movie';
|
||||||
tmdbData = await getByTmdbId(knownTmdbId, mediaType);
|
entry = await resolveEntryById(knownTmdbId, mediaType);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Try by IMDB ID
|
// Fallback: no usable id → resolve by IMDB id or fuzzy title, then cache
|
||||||
const imdbId = existingMeta?.imdbid;
|
// the result by its resolved tmdbid so later releases of the same show
|
||||||
if (!tmdbData && imdbId) {
|
// take the fast path above.
|
||||||
tmdbData = await findByImdbId(imdbId);
|
if (!entry) {
|
||||||
}
|
let tmdbData: any = null;
|
||||||
|
|
||||||
// Try by title search
|
const imdbId = existingMeta?.imdbid;
|
||||||
if (!tmdbData) {
|
if (imdbId) {
|
||||||
let title = existingMeta?.originalTitle || existingMeta?.fileName;
|
tmdbData = await findByImdbId(imdbId);
|
||||||
const year = existingMeta?.movieYear;
|
|
||||||
const videoType = existingMeta?.videoType;
|
|
||||||
|
|
||||||
// Strip trailing year from title if present (e.g., "Sintel 2010" -> "Sintel")
|
|
||||||
if (title && year) {
|
|
||||||
const yearRegex = new RegExp(`\\s*[\\(\\[]?${year}[\\)\\]]?\\s*$`);
|
|
||||||
title = title.replace(yearRegex, '').trim();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (title) {
|
if (!tmdbData) {
|
||||||
tmdbData = await searchByTitle(title, year, videoType);
|
let title = existingMeta?.originalTitle || existingMeta?.fileName;
|
||||||
|
const year = existingMeta?.movieYear;
|
||||||
|
const videoType = existingMeta?.videoType;
|
||||||
|
|
||||||
|
// Strip trailing year from title if present ("Sintel 2010" -> "Sintel").
|
||||||
|
if (title && year) {
|
||||||
|
const yearRegex = new RegExp(`\\s*[\\(\\[]?${year}[\\)\\]]?\\s*$`);
|
||||||
|
title = title.replace(yearRegex, '').trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (title) {
|
||||||
|
tmdbData = await searchByTitle(title, year, videoType);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tmdbData) {
|
||||||
|
const images = await computeImageCids(tmdbData);
|
||||||
|
entry = { data: tmdbData, ...images };
|
||||||
|
if (tmdbData.id) {
|
||||||
|
await writeJson(tmdbCacheKey(String(tmdbData.id), mediaTypeOf(tmdbData)), entry);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (tmdbData) {
|
if (entry) {
|
||||||
await applyTmdbData(metaCore, cid, tmdbData);
|
await applyTmdbData(metaCore, cid, entry.data, {
|
||||||
|
posterCid: entry.posterCid,
|
||||||
// Cache TMDB data for future use
|
backdropCid: entry.backdropCid,
|
||||||
|
});
|
||||||
|
// Keep the exact-file cache warm too, so a future re-enrichment of
|
||||||
|
// this same file skips straight to the byte-hash hit.
|
||||||
if (midhash) {
|
if (midhash) {
|
||||||
await writeJson(`${midhash}_tmdb.json`, tmdbData);
|
await writeJson(`${midhash}_tmdb.json`, entry);
|
||||||
}
|
}
|
||||||
|
console.log(`[tmdb] Enriched ${label} (tmdbid=${entry.data.id})`);
|
||||||
console.log(`[tmdb] Fetched TMDB data for ${request.filePath ?? existingMeta?.fileName ?? cid}`);
|
|
||||||
} else {
|
} else {
|
||||||
console.log(`[tmdb] No TMDB match found for ${request.filePath ?? existingMeta?.fileName ?? cid}`);
|
console.log(`[tmdb] No TMDB match found for ${label}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
await sendCallback({
|
await sendCallback({
|
||||||
@@ -534,14 +559,106 @@ export async function process(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Resolved TMDB payload plus the seeded poster/backdrop CIDs, cached per
|
||||||
|
* tmdbid. Every torrent/usenet release of an episode carries the same
|
||||||
|
* `tmdbid`, and TMDB only gives us show/movie-level fields (poster, plot,
|
||||||
|
* genres — no per-episode data), so N releases can share ONE fetch + seed. */
|
||||||
|
interface TmdbCacheEntry {
|
||||||
|
data: any;
|
||||||
|
posterCid: string | null;
|
||||||
|
backdropCid: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// In-process single-flight: collapses concurrent resolves of the same key onto
|
||||||
|
// one promise, so a burst of same-tmdbid records that all miss the cold cache
|
||||||
|
// triggers a single TMDB fetch (no thundering herd) — the rest await it.
|
||||||
|
const tmdbEntryInflight = new Map<string, Promise<TmdbCacheEntry | null>>();
|
||||||
|
|
||||||
|
// Persistent cache filename for a resolved show/movie. Includes mediaType +
|
||||||
|
// language so a tv/movie id clash or a language switch can't return stale text.
|
||||||
|
function tmdbCacheKey(tmdbId: string, mediaType: string): string {
|
||||||
|
return `tmdbid_${mediaType}_${tmdbId}_${metadataLanguage}_tmdb.json`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Media type from a raw TMDB payload's own shape (tv payloads carry
|
||||||
|
// name/first_air_date). Mirrors the check in applyTmdbData.
|
||||||
|
function mediaTypeOf(data: any): string {
|
||||||
|
return data?.first_air_date !== undefined ||
|
||||||
|
data?.name !== undefined ||
|
||||||
|
data?.original_name !== undefined
|
||||||
|
? 'tv'
|
||||||
|
: 'movie';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Download + hash the poster/backdrop for a resolved payload. Record-independent
|
||||||
|
// (the image filename is keyed by tmdbId, so the blob dedups across records), so
|
||||||
|
// the CIDs can be computed once per show and reused for every release.
|
||||||
|
async function computeImageCids(
|
||||||
|
data: any
|
||||||
|
): Promise<{ posterCid: string | null; backdropCid: string | null }> {
|
||||||
|
const tmdbId = data?.id ? String(data.id) : '';
|
||||||
|
const displayTitle =
|
||||||
|
data?.title || data?.name || data?.original_title || data?.original_name || 'Unknown';
|
||||||
|
const releaseDate = data?.release_date || data?.first_air_date;
|
||||||
|
const year = releaseDate ? releaseDate.split('-')[0] : undefined;
|
||||||
|
const posterCid =
|
||||||
|
data?.poster_path && tmdbId
|
||||||
|
? await downloadAndHashImage(data.poster_path, 'poster', displayTitle, year, tmdbId)
|
||||||
|
: null;
|
||||||
|
const backdropCid =
|
||||||
|
data?.backdrop_path && tmdbId
|
||||||
|
? await downloadAndHashImage(data.backdrop_path, 'backdrop', displayTitle, year, tmdbId)
|
||||||
|
: null;
|
||||||
|
return { posterCid, backdropCid };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Resolve the full entry (payload + image CIDs) for a KNOWN tmdbid, served from
|
||||||
|
// the persistent per-tmdbid cache when possible and single-flighted otherwise.
|
||||||
|
// This is the hot path for gateway records (torrent + usenet), which always
|
||||||
|
// carry a `tmdbid` anchor — so N releases of one show share one TMDB fetch.
|
||||||
|
async function resolveEntryById(
|
||||||
|
tmdbId: string,
|
||||||
|
mediaType: string
|
||||||
|
): Promise<TmdbCacheEntry | null> {
|
||||||
|
const key = tmdbCacheKey(tmdbId, mediaType);
|
||||||
|
if (!forceRecompute) {
|
||||||
|
const cached = await readJson<TmdbCacheEntry>(key);
|
||||||
|
if (cached?.data) {
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const inflight = tmdbEntryInflight.get(key);
|
||||||
|
if (inflight) {
|
||||||
|
return inflight;
|
||||||
|
}
|
||||||
|
const pending = (async (): Promise<TmdbCacheEntry | null> => {
|
||||||
|
console.log(`[tmdb] Resolving TMDB ${mediaType}/${tmdbId} (cache miss)`);
|
||||||
|
const data = await getByTmdbId(tmdbId, mediaType);
|
||||||
|
if (!data) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const images = await computeImageCids(data);
|
||||||
|
const entry: TmdbCacheEntry = { data, ...images };
|
||||||
|
await writeJson(key, entry);
|
||||||
|
return entry;
|
||||||
|
})();
|
||||||
|
tmdbEntryInflight.set(key, pending);
|
||||||
|
try {
|
||||||
|
return await pending;
|
||||||
|
} finally {
|
||||||
|
tmdbEntryInflight.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Apply TMDB data to KV store using same keys as old TMDBProcessor
|
* Apply TMDB data to KV store using same keys as old TMDBProcessor
|
||||||
*/
|
*/
|
||||||
async function applyTmdbData(
|
async function applyTmdbData(
|
||||||
metaCore: MetaCoreClient,
|
metaCore: MetaCoreClient,
|
||||||
cid: string,
|
cid: string,
|
||||||
data: any
|
data: any,
|
||||||
): Promise<void> {
|
images?: { posterCid: string | null; backdropCid: string | null }
|
||||||
|
): Promise<{ posterCid: string | null; backdropCid: string | null }> {
|
||||||
const metadata: Record<string, string> = {};
|
const metadata: Record<string, string> = {};
|
||||||
|
|
||||||
// Basic IDs
|
// Basic IDs
|
||||||
@@ -625,35 +742,29 @@ async function applyTmdbData(
|
|||||||
// Add tmdb-verified tag (same as old processor)
|
// Add tmdb-verified tag (same as old processor)
|
||||||
await metaCore.addToSet(cid, 'tags', 'tmdb-verified');
|
await metaCore.addToSet(cid, 'tags', 'tmdb-verified');
|
||||||
|
|
||||||
// Download poster and backdrop images to plugin output folder. The
|
// Poster/backdrop CIDs. Reuse the precomputed values when the caller already
|
||||||
// watcher picks them up as first-class file entries, so all we need to
|
// seeded them (a per-tmdbid cache hit — so we don't re-download + re-hash the
|
||||||
// store is the CID — `/api/file/{cid}` resolves through the standard
|
// same image for every release of a show); otherwise download + hash now.
|
||||||
// reverse index, no path companion needed.
|
// The image filename is keyed by tmdbId, so the blob is deduped either way —
|
||||||
if (data.poster_path && tmdbId) {
|
// this just also skips the WebDAV round-trips. The watcher picks the images
|
||||||
const posterCid = await downloadAndHashImage(
|
// up as first-class file entries, so storing the CID is all that's needed.
|
||||||
data.poster_path,
|
let posterCid = images ? images.posterCid : null;
|
||||||
'poster',
|
if (!images && data.poster_path && tmdbId) {
|
||||||
displayTitle,
|
posterCid = await downloadAndHashImage(data.poster_path, 'poster', displayTitle, year, tmdbId);
|
||||||
year,
|
}
|
||||||
tmdbId
|
if (posterCid) {
|
||||||
);
|
await metaCore.setProperty(cid, 'poster', posterCid);
|
||||||
if (posterCid) {
|
console.log(`[tmdb] Set poster CID: ${posterCid}`);
|
||||||
await metaCore.setProperty(cid, 'poster', posterCid);
|
|
||||||
console.log(`[tmdb] Set poster CID: ${posterCid}`);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (data.backdrop_path && tmdbId) {
|
let backdropCid = images ? images.backdropCid : null;
|
||||||
const backdropCid = await downloadAndHashImage(
|
if (!images && data.backdrop_path && tmdbId) {
|
||||||
data.backdrop_path,
|
backdropCid = await downloadAndHashImage(data.backdrop_path, 'backdrop', displayTitle, year, tmdbId);
|
||||||
'backdrop',
|
|
||||||
displayTitle,
|
|
||||||
year,
|
|
||||||
tmdbId
|
|
||||||
);
|
|
||||||
if (backdropCid) {
|
|
||||||
await metaCore.setProperty(cid, 'backdrop', backdropCid);
|
|
||||||
console.log(`[tmdb] Set backdrop CID: ${backdropCid}`);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
if (backdropCid) {
|
||||||
|
await metaCore.setProperty(cid, 'backdrop', backdropCid);
|
||||||
|
console.log(`[tmdb] Set backdrop CID: ${backdropCid}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return { posterCid, backdropCid };
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user