#!/usr/bin/env python3
"""Reference converter: build a Slovary dictionary pack (format v1).

Published alongside docs/mobile/PACK_FORMAT.md so that anyone can produce a
pack the app will accept. Inputs:

  * JSONL  — one {"term", "definition", "lang"} object per line
  * StarDict — .ifo (reads the sibling .idx and .dict/.dict.dz)

Standard library only, so it runs anywhere Python 3.8+ does.

    python make_pack.py --in dict.jsonl --out org.example.d.slovarypack \
        --pack-id org.example.d --name "My Dictionary" --lang en \
        --license public-domain
"""
from __future__ import annotations

import argparse
import datetime as _dt
import gzip
import html
import json
import os
import re
import sqlite3
import struct
import sys
from pathlib import Path
from typing import Iterator, Tuple

FORMAT_VERSION = 1

# Tag whitelist — MUST stay in sync with docs/mobile/PACK_FORMAT.md §4 and with
# the app-side sanitiser. Anything not listed is unwrapped (text kept, tag
# dropped); script/style/iframe-like elements are removed with their content.
ALLOWED_TAGS = {
    "b-q", "tb-q", "g-m", "t-p", "v-t", "o-u", "s-o", "f-r", "d-t",
    "e-x", "x-p", "n-m", "o-t", "r-f", "i-n",
    "b", "i", "u", "sup", "sub", "br", "b-r",
    "n-r", "b-b", "ab-r", "c-r", "s-m", "k-f", "t-r", "ya-z", "r-l",
    "s-n", "m-k", "c-b", "c-g", "e-t",
    "r", "img",
}
DROP_WITH_CONTENT = {"script", "style", "iframe", "object", "embed",
                     "link", "meta", "head", "svg"}

_TAG_RE = re.compile(r"<\s*(/?)\s*([a-zA-Z0-9\-]+)([^>]*)>")
_DROP_BLOCK_RE = re.compile(
    r"<\s*(" + "|".join(DROP_WITH_CONTENT) + r")\b.*?<\s*/\s*\1\s*>",
    re.IGNORECASE | re.DOTALL,
)
_SRC_RE = re.compile(r"""src\s*=\s*(["'])(.*?)\1""", re.IGNORECASE)


def sanitize(markup: str) -> str:
    """Drop everything outside the whitelist. Runs BEFORE the row is stored."""
    text = _DROP_BLOCK_RE.sub("", markup)
    # unclosed <script ...> with no matching end tag
    text = re.sub(r"<\s*(" + "|".join(DROP_WITH_CONTENT) + r")\b[^>]*>", "",
                  text, flags=re.IGNORECASE)

    def repl(m: "re.Match[str]") -> str:
        closing, name, attrs = m.group(1), m.group(2).lower(), m.group(3)
        if name not in ALLOWED_TAGS:
            return ""  # unwrap: keep inner text, drop the tag itself
        if name == "img":
            src = _SRC_RE.search(attrs)
            if not src:
                return ""
            url = src.group(2).strip()
            low = url.lower()
            # remote and scripted sources never reach the renderer
            if low.startswith(("http://", "https://", "//", "javascript:",
                               "data:")):
                return ""
            return '<img src="%s">' % html.escape(url, quote=True)
        return "<%s%s>" % ("/" if closing else "", name)

    return _TAG_RE.sub(repl, text).strip()


def plain_text(markup: str) -> str:
    """Markup-stripped body for the FTS index."""
    return html.unescape(re.sub(r"<[^>]+>", " ", markup))


# ---------------------------------------------------------------- readers

def read_jsonl(path: Path) -> Iterator[Tuple[str, str, str]]:
    with path.open("r", encoding="utf-8") as fh:
        for lineno, line in enumerate(fh, 1):
            line = line.strip()
            if not line:
                continue
            try:
                obj = json.loads(line)
            except json.JSONDecodeError as exc:
                raise SystemExit("%s:%d: bad JSON: %s" % (path, lineno, exc))
            term = (obj.get("term") or "").strip()
            definition = (obj.get("definition") or "").strip()
            if not term or not definition:
                continue
            yield term, definition, (obj.get("lang") or "").strip()


def _read_dict_blob(base: Path) -> bytes:
    """.dict, or .dict.dz (dictzip = gzip with random-access extras)."""
    plain, dz = base.with_suffix(".dict"), Path(str(base) + ".dict.dz")
    if plain.exists():
        return plain.read_bytes()
    if dz.exists():
        # dictzip IS gzip: the FEXTRA chunk table only matters for random
        # access, and we decompress the whole file once anyway.
        return gzip.decompress(dz.read_bytes())
    raise SystemExit("neither %s nor %s found" % (plain, dz))


def read_stardict(ifo: Path) -> Iterator[Tuple[str, str, str]]:
    meta = {}
    for line in ifo.read_text(encoding="utf-8", errors="replace").splitlines():
        if "=" in line:
            k, _, v = line.partition("=")
            meta[k.strip()] = v.strip()

    base = ifo.with_suffix("")
    idx_path = base.with_suffix(".idx")
    if not idx_path.exists():
        gz = Path(str(base) + ".idx.gz")
        if not gz.exists():
            raise SystemExit("missing %s" % idx_path)
        idx = gzip.decompress(gz.read_bytes())
    else:
        idx = idx_path.read_bytes()

    blob = _read_dict_blob(base)
    same = meta.get("sametypesequence", "")
    lang = (meta.get("bookname_lang") or "").strip()

    pos = 0
    while pos < len(idx):
        end = idx.find(b"\0", pos)
        if end < 0:
            break
        word = idx[pos:end].decode("utf-8", errors="replace")
        pos = end + 1
        if pos + 8 > len(idx):
            break
        offset, size = struct.unpack(">II", idx[pos:pos + 8])
        pos += 8
        raw = blob[offset:offset + size]
        # With sametypesequence the body carries no per-field type bytes; a
        # single-type sequence is by far the common case for real dictionaries.
        if same and len(same) == 1:
            body = raw.decode("utf-8", errors="replace")
            kind = same
        else:
            if not raw:
                continue
            kind = chr(raw[0])
            body = raw[1:].decode("utf-8", errors="replace")
        if kind in ("m", "l", "g", "h", "x", "t"):
            if kind == "m" or kind == "l" or kind == "t":
                body = html.escape(body).replace("\n", "<br>")
            yield word, body, lang
        # other field types (binary/media) are skipped


# ---------------------------------------------------------------- writer

def build(rows, out: Path, meta: dict) -> int:
    if out.exists():
        out.unlink()
    con = sqlite3.connect(str(out))
    cur = con.cursor()
    cur.execute("PRAGMA journal_mode=DELETE")
    cur.execute("CREATE TABLE pack_meta (key TEXT PRIMARY KEY, "
                "value TEXT NOT NULL)")
    cur.execute("""CREATE TABLE entries (
        local_id   INTEGER PRIMARY KEY,
        term_upper TEXT NOT NULL,
        definition TEXT NOT NULL,
        lang       TEXT NOT NULL)""")
    cur.execute("""CREATE VIRTUAL TABLE entries_fts USING fts5(
        definition_plain,
        content='',
        contentless_delete=1,
        tokenize='unicode61 remove_diacritics 2',
        detail=none)""")

    lang = meta["lang"]
    count = 0
    for term, definition, row_lang in rows:
        clean = sanitize(definition)
        if not clean:
            continue
        count += 1
        cur.execute(
            "INSERT INTO entries (local_id, term_upper, definition, lang) "
            "VALUES (?,?,?,?)",
            (count, term.upper(), clean, row_lang or lang))
        cur.execute(
            "INSERT INTO entries_fts (rowid, definition_plain) VALUES (?,?)",
            (count, plain_text(clean)))

    if count == 0:
        con.close()
        out.unlink(missing_ok=True)
        raise SystemExit("no usable entries — nothing written")

    cur.execute("CREATE INDEX idx_entries_term ON entries (term_upper)")
    meta_rows = dict(meta)
    meta_rows["format_version"] = str(FORMAT_VERSION)
    meta_rows["entries_count"] = str(count)
    meta_rows.setdefault(
        "built_at",
        _dt.datetime.now(_dt.timezone.utc).replace(microsecond=0).isoformat())
    cur.executemany("INSERT INTO pack_meta (key, value) VALUES (?,?)",
                    [(k, str(v)) for k, v in meta_rows.items() if v])
    con.commit()
    cur.execute("VACUUM")
    con.close()
    return count


def main() -> None:
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--in", dest="src", required=True,
                    help="input .jsonl or StarDict .ifo")
    ap.add_argument("--out", required=True, help="output .slovarypack")
    ap.add_argument("--pack-id", required=True,
                    help="reverse-domain id, e.g. org.gnu.gcide")
    ap.add_argument("--name", required=True, help="display name")
    ap.add_argument("--lang", required=True, help="ISO 639-1 headword language")
    ap.add_argument("--license", required=True,
                    help="SPDX id or 'public-domain'")
    ap.add_argument("--license-url", default="")
    ap.add_argument("--attribution", default="")
    ap.add_argument("--source-url", default="")
    ap.add_argument("--pack-version",
                    default=_dt.date.today().isoformat())
    args = ap.parse_args()

    src = Path(args.src)
    if not src.exists():
        raise SystemExit("input not found: %s" % src)
    rows = read_stardict(src) if src.suffix.lower() == ".ifo" \
        else read_jsonl(src)

    meta = {
        "pack_id": args.pack_id,
        "name": args.name,
        "lang": args.lang,
        "pack_version": args.pack_version,
        "license": args.license,
        "license_url": args.license_url,
        "attribution": args.attribution,
        "source_url": args.source_url,
    }
    out = Path(args.out)
    count = build(rows, out, meta)
    size_mb = out.stat().st_size / (1024 * 1024)
    print("%s: %d entries, %.1f MB" % (out.name, count, size_mb))


if __name__ == "__main__":
    sys.exit(main())