verify_posts_queued.py

"""Verify every nhentai_gallery and pornhub_video URL in a tabs file has a
corresponding task on the live server. Any missing → queue.

Dedup signals used (in order):
  1. Already on server (any status: queued/downloading/completed/failed)
     - matched by exact URL OR, for nhentai galleries, by extracted nh_id
  2. nhentai gallery: nh_id already in have_set (posts table for service='nhentai')

For pornhub videos there's no DB-side viewkey -> post.id mapping, so we rely
on the server task list as the dedup source of truth there.

Usage:
  ./.venv/bin/python verify_posts_queued.py --tabs tabs-phone2-unique-organized.txt           # report
  ./.venv/bin/python verify_posts_queued.py --tabs tabs-phone2-unique-organized.txt --queue   # POST missing
"""

from __future__ import annotations

import argparse
import json
import re
import sys
import time
import urllib.error
import urllib.request
from collections import Counter
from pathlib import Path

from gap_analysis import PG_DSN
from gap_analysis.core import load_have_set
from ingest_router import detect_route


INGEST = "http://127.0.0.1:8090/api/ingest/url"
TASKS = "http://127.0.0.1:8090/api/ingest/url/tasks"

POST_HANDLERS = {"nhentai_gallery", "pornhub_video"}
NH_ID_RE = re.compile(r"nhentai\.(?:net|xxx)/g/(\d+)")
VIEWKEY_RE = re.compile(r"viewkey=([A-Za-z0-9]+)", re.IGNORECASE)


def parse_urls(path: Path) -> list[str]:
    return [
        s
        for s in (l.strip() for l in path.read_text(encoding="utf-8", errors="replace").splitlines())
        if s and not s.startswith("=") and s.lower().startswith(("http://", "https://"))
    ]


def fetch_tasks() -> list[dict]:
    req = urllib.request.Request(TASKS, headers={"Accept": "application/json"})
    with urllib.request.urlopen(req, timeout=20) as resp:
        return json.loads(resp.read().decode("utf-8"))


def nh_id(url: str) -> int | None:
    m = NH_ID_RE.search(url)
    return int(m.group(1)) if m else None


def viewkey(url: str) -> str | None:
    m = VIEWKEY_RE.search(url)
    return m.group(1).lower() if m else None


def post(url: str, timeout: float = 15.0) -> dict:
    body = json.dumps({"url": url}).encode("utf-8")
    req = urllib.request.Request(
        INGEST, data=body, method="POST",
        headers={"Content-Type": "application/json", "Accept": "application/json"},
    )
    with urllib.request.urlopen(req, timeout=timeout) as resp:
        return json.loads(resp.read().decode("utf-8"))


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--tabs", required=True)
    ap.add_argument("--queue", action="store_true")
    ap.add_argument("--delay", type=float, default=0.15)
    args = ap.parse_args()

    p = Path(args.tabs)
    if not p.exists():
        print(f"not found: {p}", file=sys.stderr); return 2

    urls = parse_urls(p)
    print(f"[1/4] {len(urls)} url lines in {p.name}", file=sys.stderr)

    targets: list[tuple[str, str]] = []  # (url, handler)
    for u in urls:
        r = detect_route(u)
        if r and r.handler in POST_HANDLERS:
            targets.append((u, r.handler))
    print(f"[2/4] {len(targets)} are nhentai_gallery or pornhub_video", file=sys.stderr)

    print("[3/4] fetching server tasks + nhentai have-set", file=sys.stderr)
    tasks = fetch_tasks()
    have_set = load_have_set(PG_DSN)
    print(f"      server tasks: {len(tasks)}  |  nhentai have-set: {len(have_set)} nh_ids", file=sys.stderr)

    # Index server tasks for fast lookup
    task_urls: set[str] = {t.get("url") for t in tasks if t.get("url")}
    task_nh_ids: set[int] = set()
    task_viewkeys: set[str] = set()
    for t in tasks:
        u = t.get("url", "")
        n = nh_id(u)
        if n is not None:
            task_nh_ids.add(n)
        vk = viewkey(u)
        if vk:
            task_viewkeys.add(vk.lower())

    missing: list[tuple[str, str]] = []
    reasons: Counter[str] = Counter()

    for u, h in targets:
        if u in task_urls:
            reasons["on-server-exact-url"] += 1
            continue
        if h == "nhentai_gallery":
            n = nh_id(u)
            if n is not None and n in have_set:
                reasons["nhentai-in-db"] += 1
                continue
            if n is not None and n in task_nh_ids:
                reasons["nhentai-task-by-id"] += 1
                continue
        elif h == "pornhub_video":
            vk = viewkey(u)
            if vk and vk in task_viewkeys:
                reasons["pornhub-task-by-viewkey"] += 1
                continue
        missing.append((u, h))

    print(f"[4/4] coverage report:", file=sys.stderr)
    for r, n in reasons.most_common():
        print(f"      {r:30s} {n}", file=sys.stderr)
    print(f"      MISSING:                       {len(missing)}", file=sys.stderr)
    if missing:
        miss_by_handler = Counter(h for _, h in missing)
        for h, n in miss_by_handler.most_common():
            print(f"        {h:25s} {n}", file=sys.stderr)

    if not args.queue or not missing:
        for u, h in missing[:30]:
            print(f"{h:20s} {u}")
        if len(missing) > 30:
            print(f"... and {len(missing) - 30} more")
        return 0

    print(f"\nqueueing {len(missing)} missing URLs", file=sys.stderr)
    ok = fail = dup = 0
    for i, (u, h) in enumerate(missing, 1):
        try:
            r = post(u)
            if r.get("duplicate"):
                dup += 1; tag = "DUP"
            else:
                ok += 1; tag = "OK "
            print(f"[{i:3d}/{len(missing)}] {tag} {h:20s} {(r.get('task_id','-') or '-')[:8]} {u}")
        except Exception as e:  # noqa: BLE001
            fail += 1
            print(f"[{i:3d}/{len(missing)}] ERR {h:20s} {e}  {u}", file=sys.stderr)
        if args.delay > 0 and i < len(missing):
            time.sleep(args.delay)

    print(f"\ndone: ok={ok} dup={dup} fail={fail}", file=sys.stderr)
    return 0 if fail == 0 else 1


if __name__ == "__main__":
    sys.exit(main())