"""Search parity check for meta-core search.

  parity.py <cache.db>

Since v2.0.0 the searchable records are the MERGED view, kept in the cache
database's kv table under the v1 layout (file:<rid>/<field>, idx:roots/<rid>):
point this at /meta-core/db-cache/cache.db, not the truth database.

Builds a query set from real records, computes the EXPECTED result of each
query by brute force — one streaming pass over every record in the database,
evaluated with a line-by-line port of the Go matchers (api.recordMatchesFilter,
api.matchAllTokens, storage.Haystack) — and diffs it against what the live
POST /api/metadata/search returns. A miss means the SQL candidate superset
dropped a real match; that is the failure this exists to catch.
"""
import json, random, sqlite3, sys, time, urllib.request

BASE = "http://127.0.0.1:9000"
LIMIT = 2000
DB = sys.argv[1]


def post(body):
    req = urllib.request.Request(BASE + "/api/metadata/search", data=json.dumps(body).encode(),
                                 headers={"Content-Type": "application/json"})
    t0 = time.time()
    with urllib.request.urlopen(req, timeout=300) as resp:
        data = json.load(resp)
    return [r["hashId"] for r in data["results"]], time.time() - t0


SEARCH_FIELDS = ["title", "fileName", "originaltitle", "showtitle", "filePath"]


def haystack(f):
    parts = [f[k].lower() for k in SEARCH_FIELDS if f.get(k)]
    names = []
    for k, v in f.items():
        if k.startswith("titles/") and v not in ("", "false"):
            rest = k[len("titles/"):]
            if "/" in rest:
                name = rest.split("/", 1)[1]
                if name:
                    names.append(name.replace("∕", "/"))
    parts += [n.lower() for n in sorted(names)]
    return "".join(p + "\n" for p in parts)


def truthy(v):
    v = (v or "").strip().lower()
    return v not in ("", "false", "0")


def match_filter(f, flt):
    values = [v.strip() for v in flt.get("values", []) if v.strip()]
    prop = flt["property"]
    if not prop or not values:
        return True
    for v in values:
        if truthy(f.get(prop + "/" + v)):
            return True
    wanted = {v.lower() for v in values}
    prefix = prop.lower() + "/"
    for k, val in f.items():
        if len(k) > len(prefix) and k[:len(prefix)].lower() == prefix:
            if k[len(prefix):].lower() in wanted and truthy(val):
                return True
    if prop not in f:
        return False
    for piece in f[prop].split(","):
        piece = piece.strip()
        if not piece:
            continue
        for v in values:
            if flt.get("exact"):
                if piece.lower() == v.lower():
                    return True
            elif v.lower() in piece.lower():
                return True
    return False


def match(rid, f, q):
    query = q.get("query", "")
    if query:
        tokens = query.lower().split()
        hay = haystack(f)
        if not all(t in hay for t in tokens) and query.lower() not in rid.lower():
            return False
    return all(match_filter(f, flt) for flt in q.get("filters", []))


def records(db):
    cur = db.execute("SELECT k, v FROM kv WHERE k >= 'file:' AND k < 'file;' ORDER BY k")
    rid, fields = None, {}
    for k, v in cur:
        r, sep, field = k[5:].partition("/")
        if not sep or not field:
            continue
        if r != rid:
            if rid is not None:
                yield rid, fields
            rid, fields = r, {}
        fields[field] = v
    if rid is not None:
        yield rid, fields


def build_queries(db):
    random.seed(7)
    roots = [r for (r,) in db.execute("SELECT substr(k, 11) FROM kv WHERE k >= 'idx:roots/' AND k < 'idx:roots0'")]
    sample = []
    for rid in random.sample(roots, 80):
        f = {k[len(rid) + 6:]: v for k, v in db.execute(
            "SELECT k, v FROM kv WHERE k >= ? AND k < ?", (f"file:{rid}/", f"file:{rid}0"))}
        sample.append(f)
    qs = [{}]
    for fld in ("fileType", "contentKind", "domain", "workForm", "videoType", "provenance", "quality", "indexer"):
        for v in sorted({m[fld] for m in sample if m.get(fld)})[:3]:
            qs.append({"filters": [{"property": fld, "values": [v], "exact": True}]})
            qs.append({"filters": [{"property": fld, "values": [v.upper()[:3]]}]})
    for m in sample[:30]:
        for k in sorted(m):
            if k.startswith(("genres/", "categories/", "source/", "titles/")) and k.count("/") == 1:
                p, v = k.split("/", 1)
                qs.append({"filters": [{"property": p, "values": [v.lower()]}]})
                break
        for idf in ("tmdbid", "imdbid", "season"):
            if m.get(idf):
                qs.append({"filters": [{"property": idf, "values": [m[idf]], "exact": True}]})
        words = (m.get("title") or "").split()
        if words:
            w = max(words, key=len)
            qs.append({"query": w})
            qs.append({"query": w[:2]})
            qs.append({"query": " ".join(words[:2])})
            if m.get("fileType"):
                qs.append({"query": w, "filters": [{"property": "fileType", "values": [m["fileType"]]}]})
    for m in sample[30:45]:
        words = (m.get("title") or "").split()
        if words and m.get("contentKind"):
            qs.append({"query": max(words, key=len), "filters": [{"property": "contentKind", "values": [m["contentKind"]], "exact": True}]})
            qs.append({"query": max(words, key=len), "filters": [{"property": "fileType", "values": ["video"]}, {"property": "contentKind", "values": [m["contentKind"]]}]})
    qs += [{"query": "s01e01"}, {"query": "1080p", "filters": [{"property": "fileType", "values": ["video"], "exact": True}]},
           {"filters": [{"property": "genres", "values": ["action", "comedy"]}, {"property": "fileType", "values": ["video"], "exact": True}]}]
    for q in qs:
        q["limit"] = LIMIT
    return qs


def main():
    db = sqlite3.connect(f"file:{DB}?mode=ro", uri=True)
    qs = build_queries(db)
    print(f"{len(qs)} queries; computing reference over every record...", flush=True)
    t0 = time.time()
    expected = [[] for _ in qs]
    n = 0
    for rid, f in records(db):
        n += 1
        for i, q in enumerate(qs):
            if match(rid, f, q):
                expected[i].append(rid)
    print(f"reference: {n} records in {time.time() - t0:.0f}s", flush=True)

    bad = 0
    for q, exp in zip(qs, expected):
        exp = sorted(exp)[:LIMIT]
        got, secs = post(q)
        lost, extra = set(exp) - set(got), set(got) - set(exp)
        order_ok = got == sorted(got)
        status = "OK  " if not lost and not extra and order_ok else "DIFF"
        bad += status != "OK  "
        print(f"{status} want={len(exp):5d} got={len(got):5d} lost={len(lost):4d} extra={len(extra):4d} "
              f"{secs*1000:7.1f}ms  {json.dumps({k: v for k, v in q.items() if k != 'limit'})[:100]}", flush=True)
        if lost:
            print("     lost e.g.", sorted(lost)[:4])
        if extra:
            print("     extra e.g.", sorted(extra)[:4])
    print(f"\n{bad} of {len(qs)} queries differ")


main()