#!/usr/bin/env python3
"""Convert GCIDE source files (CIDE.A … CIDE.Z) to pack-builder JSONL.

GCIDE = GNU Collaborative International Dictionary of English: the
public-domain Webster's 1913 plus volunteer and WordNet additions, released
under GPL-3.0-or-later. https://gcide.gnu.org.ua/

Usage:
    python gcide_to_jsonl.py --src /path/to/gcide-0.54 --out gcide.jsonl

Then feed the result to make_pack.py.

Format notes (learned from the 0.54 sources):
  * An entry is a run of <p>…</p> blocks: the block carrying <ent> starts it,
    following blocks WITHOUT <ent> are extra senses of the same headword.
  * Tags are SGML-ish and often unclosed (`<br/` with no `>`), so a real XML
    parser is not usable — this is a targeted text transform.
  * Diacritics are spelled as pseudo-tags (`<amac/` = a-macron). Most of them
    live inside <pr> pronunciation blocks, which are dropped: without the
    project's own webfont they render as noise.
"""
from __future__ import annotations

import argparse
import html
import json
import re
import sys
from pathlib import Path

# --- pseudo-tag character entities -------------------------------------
# Only the forms that actually survive outside <pr>. Unknown `<xxx/` forms
# are deleted rather than guessed at.
ENTITIES = {
    "ldquo": "“", "rdquo": "”", "lsquo": "‘", "rsquo": "’",
    "ae": "æ", "AE": "Æ", "oe": "œ", "OE": "Œ",
    "eacute": "é", "egrave": "è", "ecirc": "ê",
    "aacute": "á", "agrave": "à", "acirc": "â",
    "iacute": "í", "oacute": "ó", "uacute": "ú",
    "ocirc": "ô", "ucirc": "û", "icirc": "î",
    "aum": "ä", "oum": "ö", "uum": "ü", "eum": "ë",
    "ium": "ï", "yum": "ÿ",
    "ntil": "ñ", "ccedil": "ç", "edh": "ð", "thorn": "þ",
    "osl": "ø", "aring": "å",
    "amac": "ā", "emac": "ē", "imac": "ī",
    "omac": "ō", "umac": "ū",
    "acr": "ă", "ecr": "ĕ", "icr": "ĭ",
    "ocr": "ŏ", "ucr": "ŭ", "ycr": "y̆",
    "adot": "ȧ", "etil": "ẽ", "atil": "ã", "otil": "õ",
    "hand": "☞", "root": "√", "deg": "°",
    "sect": "§", "pound": "£", "cent": "¢",
    "frac12": "½", "frac14": "¼", "frac34": "¾",
    "prime": "′", "dagger": "†", "Dagger": "‡",
    "mdash": "—", "ndash": "–", "hellip": "…",
    "times": "×", "divide": "÷", "plusmn": "±",
    "alpha": "α", "beta": "β", "gamma": "γ", "delta": "δ",
    "or": " or ", "add": "", "omit": "",
}

# GCIDE tag -> our tag (see docs/mobile/PACK_FORMAT.md §4).
BLOCK_MAP = {
    "def": "b-q",     # the definition itself
    "ety": "d-t",     # etymology
    "syn": "f-r",     # synonyms
    "note": "n-m",    # editorial note
    "cd": "tb-q",     # collocation definition
}
INLINE_MAP = {
    "pos": "s-n",     # part of speech
    "fld": "s-n",     # subject field (Chem., Naut., …)
    "mark": "s-n",
    "b": "b", "i": "i", "u": "u",
    "col": "n-r",     # collocation headword
    "as": "i", "ex": "i", "xex": "i", "qex": "i", "it": "i",
    "chform": "i", "conjf": "i", "plw": "i", "singw": "i",
    "ets": "i",       # etymon spelling — keep, it carries the etymology text
}
# Dropped with their content: pronunciation needs the project webfont;
# <hw> duplicates <ent> with syllable dots; the rest is apparatus.
#
# <er> (entry reference) is deliberately NOT here: it names another headword,
# so it maps to <r> and becomes a working cross-link in the app. Dropping it
# also left etymologies full of holes like "[Cf. F. . See .]".
DROP_CONTENT = ("pr", "hw", "ent", "wf", "wns", "altsp",
                "altname", "asp", "amorph", "abbr", "stype", "cs")

_P_RE = re.compile(r"<p>(.*?)</p>", re.DOTALL)
_ENT_RE = re.compile(r"<ent>(.*?)</ent>", re.DOTALL)
_ENTITY_RE = re.compile(r"<([a-zA-Z0-9]+)/")
# Leftover glyph pseudo-tags: `<amac/`, `<e"/`, `<frac12/` … They never start
# with `/` and never contain whitespace — WITHOUT the `(?!/)` guard this also
# ate the `</` of every closing tag, turning `</pos>` into a literal `pos>`.
_ANY_SELFCLOSE_RE = re.compile(r"<(?!/)[^<>\s]{1,16}/(?!>)")
_TAG_RE = re.compile(r"</?([a-zA-Z0-9]+)[^>]*>")
# Provenance marker of a sense, e.g. "[<source>1913 Webster</source>]". Only
# the bracketed form is a provenance marker — <r-f> is also what a quotation
# attribution ("Shak.", "Milton.") becomes, and that one belongs inline.
_SRC_MARK_RE = re.compile(r"\[\s*<r-f>([^\[\]]*?)</r-f>\s*\]", re.DOTALL)
_SRC_DUP_RE = re.compile(r"(<r-f>\[[^\[\]]*\]</r-f><br>)(?:\s*\1)+")


def _strip_drop_content(text: str) -> str:
    for tag in DROP_CONTENT:
        text = re.sub(r"<%s>.*?</%s>" % (tag, tag), " ", text,
                      flags=re.DOTALL)
    return text


def convert_body(raw: str) -> str:
    """GCIDE block markup -> the pack format's restricted tag set."""
    text = raw.replace("<br/", "<br>")
    text = _strip_drop_content(text)

    # pseudo-tag entities first, then delete whatever self-closing forms are
    # left (unknown glyph names — dropping beats emitting mojibake)
    text = _ENTITY_RE.sub(
        lambda m: ENTITIES.get(m.group(1), "\x00"), text)
    text = text.replace("\x00", "")
    text = _ANY_SELFCLOSE_RE.sub("", text)

    def repl(m: "re.Match[str]") -> str:
        whole, name = m.group(0), m.group(1).lower()
        closing = whole.startswith("</")
        if name == "br":
            return "<br>"
        if name in BLOCK_MAP:
            return "</%s>" % BLOCK_MAP[name] if closing \
                else "<%s>" % BLOCK_MAP[name]
        if name in INLINE_MAP:
            return "</%s>" % INLINE_MAP[name] if closing \
                else "<%s>" % INLINE_MAP[name]
        if name in ("q", "quote"):          # cited quotation
            return "</e-x>" if closing else "<e-x>"
        if name in ("qau", "au", "source"):  # attribution / source
            return "</r-f>" if closing else "<r-f>"
        if name == "sn":                     # sense number
            return "</b>" if closing else "<b>"
        if name in ("er", "cref"):           # reference to another headword
            return "</r>" if closing else "<r>"
        return ""                            # unknown tag: unwrap

    text = _TAG_RE.sub(repl, text)
    # dropped <pr>/<hw> leave orphaned punctuation and empty brackets behind
    text = re.sub(r"\[\s*([.,;:]\s*)*\]", "", text)
    text = re.sub(r"\s+([,;.])(\s*[,;.])+", r"\1", text)
    # The provenance marker's brackets are plain text around the tag, and
    # nothing ends the line after them; <r-f> renders inline, so the marker
    # glued itself to the next sense number and every multi-sense article read
    # "[1913 Webster]2. A calculating table…". Pull the brackets inside the
    # tag and close the line, then drop the marker repeated back-to-back
    # (which is how GCIDE ends many entries).
    text = _SRC_MARK_RE.sub(r"<r-f>[\1]</r-f><br>", text)
    text = _SRC_DUP_RE.sub(r"\1", text)
    text = re.sub(r"(<br>\s*){3,}", "<br><br>", text)
    text = re.sub(r"[ \t]{2,}", " ", text)
    text = re.sub(r"^(?:\s|<br>|[,;:.]|&nbsp;)+", "", text)
    text = re.sub(r"(?:\s*<br>)+$", "", text)
    return text.strip()


def parse_file(path: Path):
    """Yields (headword, body_html) — continuation blocks fold into the entry.

    Blocks are joined with an explicit <br>: a continuation block is the next
    SENSE of the headword, and concatenating them bare ran the provenance
    marker that ends one block into the sense number that opens the next
    ("[1913 Webster]2. A calculating table…" in every multi-sense article).
    """
    text = path.read_text(encoding="utf-8", errors="replace")
    current_word = None
    parts: list[str] = []
    for block in _P_RE.finditer(text):
        raw = block.group(1)
        ents = _ENT_RE.findall(raw)
        body = convert_body(raw)
        if ents:
            if current_word and parts:
                yield current_word, "<br>".join(parts)
            current_word = html.unescape(ents[0]).strip()
            parts = [body] if body else []
        elif current_word and body:
            parts.append(body)
    if current_word and parts:
        yield current_word, "".join(parts)


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--src", required=True, help="unpacked gcide-X.YZ directory")
    ap.add_argument("--out", required=True, help="output .jsonl")
    args = ap.parse_args()

    src = Path(args.src)
    files = sorted(p for p in src.glob("CIDE.*") if len(p.suffix) == 2)
    if not files:
        raise SystemExit("no CIDE.* files under %s" % src)

    written = skipped = 0
    with Path(args.out).open("w", encoding="utf-8") as out:
        for path in files:
            for word, body in parse_file(path):
                if not word or not body or len(body) < 12:
                    skipped += 1
                    continue
                out.write(json.dumps(
                    {"term": word, "definition": body, "lang": "en"},
                    ensure_ascii=False) + "\n")
                written += 1
            print("  %s done (%d so far)" % (path.name, written),
                  file=sys.stderr)
    print("wrote %d entries, skipped %d" % (written, skipped))
    return 0


if __name__ == "__main__":
    sys.exit(main())