179 lines
6.8 KiB
Python
179 lines
6.8 KiB
Python
"""Search parity check for the meta-core Redis -> SQLite cut-over.
|
||
|
||
parity.py <meta-core.db>
|
||
|
||
Builds a query set from real records, computes the EXPECTED result of each
|
||
query by brute force — one streaming pass over every record in the database,
|
||
evaluated with a line-by-line port of the Go matchers (api.recordMatchesFilter,
|
||
api.matchAllTokens, storage.Haystack) — and diffs it against what the live
|
||
POST /api/metadata/search returns. A miss means the SQL candidate superset
|
||
dropped a real match; that is the failure this exists to catch.
|
||
"""
|
||
import json, random, sqlite3, sys, time, urllib.request
|
||
|
||
BASE = "http://127.0.0.1:9000"
|
||
LIMIT = 2000
|
||
DB = sys.argv[1]
|
||
|
||
|
||
def post(body):
|
||
req = urllib.request.Request(BASE + "/api/metadata/search", data=json.dumps(body).encode(),
|
||
headers={"Content-Type": "application/json"})
|
||
t0 = time.time()
|
||
with urllib.request.urlopen(req, timeout=300) as resp:
|
||
data = json.load(resp)
|
||
return [r["hashId"] for r in data["results"]], time.time() - t0
|
||
|
||
|
||
SEARCH_FIELDS = ["title", "fileName", "originaltitle", "showtitle", "filePath"]
|
||
|
||
|
||
def haystack(f):
|
||
parts = [f[k].lower() for k in SEARCH_FIELDS if f.get(k)]
|
||
names = []
|
||
for k, v in f.items():
|
||
if k.startswith("titles/") and v not in ("", "false"):
|
||
rest = k[len("titles/"):]
|
||
if "/" in rest:
|
||
name = rest.split("/", 1)[1]
|
||
if name:
|
||
names.append(name.replace("∕", "/"))
|
||
parts += [n.lower() for n in sorted(names)]
|
||
return "".join(p + "\n" for p in parts)
|
||
|
||
|
||
def truthy(v):
|
||
v = (v or "").strip().lower()
|
||
return v not in ("", "false", "0")
|
||
|
||
|
||
def match_filter(f, flt):
|
||
values = [v.strip() for v in flt.get("values", []) if v.strip()]
|
||
prop = flt["property"]
|
||
if not prop or not values:
|
||
return True
|
||
for v in values:
|
||
if truthy(f.get(prop + "/" + v)):
|
||
return True
|
||
wanted = {v.lower() for v in values}
|
||
prefix = prop.lower() + "/"
|
||
for k, val in f.items():
|
||
if len(k) > len(prefix) and k[:len(prefix)].lower() == prefix:
|
||
if k[len(prefix):].lower() in wanted and truthy(val):
|
||
return True
|
||
if prop not in f:
|
||
return False
|
||
for piece in f[prop].split(","):
|
||
piece = piece.strip()
|
||
if not piece:
|
||
continue
|
||
for v in values:
|
||
if flt.get("exact"):
|
||
if piece.lower() == v.lower():
|
||
return True
|
||
elif v.lower() in piece.lower():
|
||
return True
|
||
return False
|
||
|
||
|
||
def match(rid, f, q):
|
||
query = q.get("query", "")
|
||
if query:
|
||
tokens = query.lower().split()
|
||
hay = haystack(f)
|
||
if not all(t in hay for t in tokens) and query.lower() not in rid.lower():
|
||
return False
|
||
return all(match_filter(f, flt) for flt in q.get("filters", []))
|
||
|
||
|
||
def records(db):
|
||
cur = db.execute("SELECT k, v FROM kv WHERE k >= 'file:' AND k < 'file;' ORDER BY k")
|
||
rid, fields = None, {}
|
||
for k, v in cur:
|
||
r, sep, field = k[5:].partition("/")
|
||
if not sep or not field:
|
||
continue
|
||
if r != rid:
|
||
if rid is not None:
|
||
yield rid, fields
|
||
rid, fields = r, {}
|
||
fields[field] = v
|
||
if rid is not None:
|
||
yield rid, fields
|
||
|
||
|
||
def build_queries(db):
|
||
random.seed(7)
|
||
roots = [r for (r,) in db.execute("SELECT substr(k, 11) FROM kv WHERE k >= 'idx:roots/' AND k < 'idx:roots0'")]
|
||
sample = []
|
||
for rid in random.sample(roots, 80):
|
||
f = {k[len(rid) + 6:]: v for k, v in db.execute(
|
||
"SELECT k, v FROM kv WHERE k >= ? AND k < ?", (f"file:{rid}/", f"file:{rid}0"))}
|
||
sample.append(f)
|
||
qs = [{}]
|
||
for fld in ("fileType", "contentKind", "domain", "workForm", "videoType", "provenance", "quality", "indexer"):
|
||
for v in sorted({m[fld] for m in sample if m.get(fld)})[:3]:
|
||
qs.append({"filters": [{"property": fld, "values": [v], "exact": True}]})
|
||
qs.append({"filters": [{"property": fld, "values": [v.upper()[:3]]}]})
|
||
for m in sample[:30]:
|
||
for k in sorted(m):
|
||
if k.startswith(("genres/", "categories/", "source/", "titles/")) and k.count("/") == 1:
|
||
p, v = k.split("/", 1)
|
||
qs.append({"filters": [{"property": p, "values": [v.lower()]}]})
|
||
break
|
||
for idf in ("tmdbid", "imdbid", "season"):
|
||
if m.get(idf):
|
||
qs.append({"filters": [{"property": idf, "values": [m[idf]], "exact": True}]})
|
||
words = (m.get("title") or "").split()
|
||
if words:
|
||
w = max(words, key=len)
|
||
qs.append({"query": w})
|
||
qs.append({"query": w[:2]})
|
||
qs.append({"query": " ".join(words[:2])})
|
||
if m.get("fileType"):
|
||
qs.append({"query": w, "filters": [{"property": "fileType", "values": [m["fileType"]]}]})
|
||
for m in sample[30:45]:
|
||
words = (m.get("title") or "").split()
|
||
if words and m.get("contentKind"):
|
||
qs.append({"query": max(words, key=len), "filters": [{"property": "contentKind", "values": [m["contentKind"]], "exact": True}]})
|
||
qs.append({"query": max(words, key=len), "filters": [{"property": "fileType", "values": ["video"]}, {"property": "contentKind", "values": [m["contentKind"]]}]})
|
||
qs += [{"query": "s01e01"}, {"query": "1080p", "filters": [{"property": "fileType", "values": ["video"], "exact": True}]},
|
||
{"filters": [{"property": "genres", "values": ["action", "comedy"]}, {"property": "fileType", "values": ["video"], "exact": True}]}]
|
||
for q in qs:
|
||
q["limit"] = LIMIT
|
||
return qs
|
||
|
||
|
||
def main():
|
||
db = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
|
||
qs = build_queries(db)
|
||
print(f"{len(qs)} queries; computing reference over every record...", flush=True)
|
||
t0 = time.time()
|
||
expected = [[] for _ in qs]
|
||
n = 0
|
||
for rid, f in records(db):
|
||
n += 1
|
||
for i, q in enumerate(qs):
|
||
if match(rid, f, q):
|
||
expected[i].append(rid)
|
||
print(f"reference: {n} records in {time.time() - t0:.0f}s", flush=True)
|
||
|
||
bad = 0
|
||
for q, exp in zip(qs, expected):
|
||
exp = sorted(exp)[:LIMIT]
|
||
got, secs = post(q)
|
||
lost, extra = set(exp) - set(got), set(got) - set(exp)
|
||
order_ok = got == sorted(got)
|
||
status = "OK " if not lost and not extra and order_ok else "DIFF"
|
||
bad += status != "OK "
|
||
print(f"{status} want={len(exp):5d} got={len(got):5d} lost={len(lost):4d} extra={len(extra):4d} "
|
||
f"{secs*1000:7.1f}ms {json.dumps({k: v for k, v in q.items() if k != 'limit'})[:100]}", flush=True)
|
||
if lost:
|
||
print(" lost e.g.", sorted(lost)[:4])
|
||
if extra:
|
||
print(" extra e.g.", sorted(extra)[:4])
|
||
print(f"\n{bad} of {len(qs)} queries differ")
|
||
|
||
|
||
main()
|