#!/usr/bin/env python3
"""State of African Data Protection Enforcement, H1 2026 — analysis script.

Single source of every statistic in the report. Reads the data spine
(products/research_wing/data/), writes:

  figures.json                        every number the report may cite
  charts/*.svg + *.png                branded charts (matplotlib, lawlab.mplstyle)
  state-of-enforcement-h1-2026.md    rendered from the .src.md template

No statistic is hand-typed in the report: the template carries {{tokens}} and
this script substitutes them, so figures regenerate whenever the datasets
change. Charts need matplotlib (use a venv: /tmp/env on W's machine); the
figures/report render runs on stdlib only.

Usage:  python3 analyze.py            # figures + report render
        /tmp/env/bin/python analyze.py --charts   # also rebuild charts
"""

import json
import re
import statistics
import sys
from collections import Counter
from datetime import date
from pathlib import Path

HERE = Path(__file__).resolve().parent
DATA = HERE.parent / "data"
CHARTS = HERE / "charts"
TEMPLATE = HERE / "state-of-enforcement-h1-2026.src.md"
REPORT = HERE / "state-of-enforcement-h1-2026.md"
BRAND_STYLE = HERE.parent.parent.parent / "brand" / "charts" / "lawlab.mplstyle"

H1_LO, H1_HI = "2026-01", "2026-06"


def load():
    decisions = json.loads((DATA / "decisions.json").read_text())
    landscape = json.loads((DATA / "landscape.json").read_text())
    return decisions, landscape


def ym(rec):
    return str(rec["date_decided"])[:7]


def quarter(rec):
    y, m = ym(rec).split("-")
    return f"{y}-Q{(int(m) - 1) // 3 + 1}"


def fmt_int(n):
    return f"{n:,}"


def fmt_usd(n):
    return f"USD {round(n):,}"


def pct(part, whole):
    return f"{round(100 * part / whole)}%"


def compute(decisions, landscape):
    f = {}
    today = date.today().isoformat()
    f["build_date"] = today
    f["build_year"] = today[:4]

    # ------------------------------------------------------------- dataset
    f["n_total"] = fmt_int(len(decisions))
    juris = Counter(x["country"] for x in decisions)
    f["n_jurisdictions"] = len(juris)
    for c, n in juris.items():
        f[f"n_{c.lower().replace(' ', '_')}"] = fmt_int(n)
    f["pct_kenya"] = pct(juris["Kenya"], len(decisions))
    years = sorted({ym(x)[:4] for x in decisions})
    f["year_min"], f["year_max"] = years[0], years[-1]
    f["n_month_precision"] = fmt_int(
        sum(1 for x in decisions if x.get("date_precision") == "month"))
    f["n_verified"] = fmt_int(
        sum(1 for x in decisions if x["confidence"] == "verified"))
    f["n_fsca"] = fmt_int(sum(1 for x in decisions if x["regulator"] == "FSCA"))

    dtypes = Counter(x["decision_type"] for x in decisions)
    for k, v in dtypes.items():
        f[f"n_type_{k}"] = fmt_int(v)

    # ------------------------------------------------------------- H1 2026
    h1 = [x for x in decisions if H1_LO <= ym(x) <= H1_HI]
    f["n_h1_2026"] = fmt_int(len(h1))
    h1j = Counter(x["country"] for x in h1)
    f["h1_jurisdictions"] = ", ".join(
        f"{c} {n}" for c, n in h1j.most_common())
    f["n_h1_kenya"] = fmt_int(h1j.get("Kenya", 0))
    f["n_h1_2025"] = fmt_int(  # same window last year, for comparison
        sum(1 for x in decisions if "2025-01" <= ym(x) <= "2025-06"))
    f["n_2025_full"] = fmt_int(sum(1 for x in decisions if ym(x)[:4] == "2025"))
    f["n_2024_full"] = fmt_int(sum(1 for x in decisions if ym(x)[:4] == "2024"))
    f["n_2023_full"] = fmt_int(sum(1 for x in decisions if ym(x)[:4] == "2023"))
    f["n_h1_q1"] = fmt_int(sum(1 for x in h1 if quarter(x) == "2026-Q1"))
    f["n_h1_q2"] = fmt_int(sum(1 for x in h1 if quarter(x) == "2026-Q2"))

    # ------------------------------------------------------------ landscape
    dp_inforce = [c for c in landscape
                  if (c.get("dp_law") or {}).get("status") == "in_force"]
    f["n_countries"] = fmt_int(len(landscape))
    f["n_laws_inforce"] = fmt_int(len(dp_inforce))
    f["n_laws_bill"] = fmt_int(sum(
        1 for c in landscape if (c.get("dp_law") or {}).get("status") == "bill"))
    f["n_laws_enacted_nc"] = fmt_int(sum(
        1 for c in landscape
        if (c.get("dp_law") or {}).get("status") == "enacted_not_commenced"))
    f["n_laws_none"] = fmt_int(sum(
        1 for c in landscape if not c.get("dp_law")
        or (c.get("dp_law") or {}).get("status") == "none"))
    f["n_reg_operational"] = fmt_int(sum(
        1 for c in landscape if (c.get("regulator") or {}).get("operational")))
    f["n_enforcing"] = fmt_int(
        sum(1 for c in landscape if c.get("enforcement_active")))
    f["n_publishing"] = fmt_int(
        sum(1 for c in landscape if c.get("decisions_published")))
    enf_not_pub = sorted(c["country"] for c in landscape
                         if c.get("enforcement_active")
                         and not c.get("decisions_published"))
    f["enforcing_not_publishing"] = ", ".join(enf_not_pub)
    f["n_enforcing_not_publishing"] = fmt_int(len(enf_not_pub))
    nonen = [c for c in landscape if c.get("decisions_published")
             and c.get("publication_language") not in (None, "en")]
    f["n_publishing_non_english"] = fmt_int(len(nonen))
    f["tier1_countries"] = ", ".join(sorted(
        c["country"] for c in landscape if c["tracker_status"] == "tier1"))
    f["tier2_countries"] = ", ".join(sorted(
        c["country"] for c in landscape if c["tracker_status"] == "tier2"))
    f["n_tier2"] = fmt_int(
        sum(1 for c in landscape if c["tracker_status"] == "tier2"))

    # ------------------------------------------------------------- money
    pens = [x for x in decisions if x.get("penalty_usd")]
    f["n_penalties"] = fmt_int(len(pens))
    pu = sorted(x["penalty_usd"] for x in pens)
    f["penalty_median_usd"] = fmt_usd(statistics.median(pu))
    f["penalty_mean_usd"] = fmt_usd(statistics.mean(pu))
    f["penalty_max_usd"] = fmt_usd(pu[-1])
    pu_ex_meta = [v for v in pu if v < 30_000_000]  # exclude both Meta fines
    f["penalty_mean_usd_ex_meta"] = fmt_usd(statistics.mean(pu_ex_meta))
    f["penalty_median_usd_ex_meta"] = fmt_usd(statistics.median(pu_ex_meta))

    comps = [x for x in decisions if x.get("compensation_usd")]
    f["n_compensation"] = fmt_int(len(comps))
    cu = sorted(x["compensation_usd"] for x in comps)
    ck = sorted(x["compensation_amount"] for x in comps)
    f["comp_median_usd"] = fmt_usd(statistics.median(cu))
    f["comp_mean_usd"] = fmt_usd(statistics.mean(cu))
    f["comp_median_kes"] = f"KES {round(statistics.median(ck)):,}"
    f["comp_max_kes"] = f"KES {max(ck):,}"
    f["comp_max_usd"] = fmt_usd(max(cu))
    f["comp_total_kes"] = f"KES {sum(ck):,}"
    f["comp_total_usd"] = fmt_usd(sum(cu))

    # ------------------------------------------------------- outcome flavour
    upheld = [x for x in decisions if "complaint upheld" in x["outcome"]]
    dismissed = [x for x in decisions if "complaint dismissed" in x["outcome"]]
    f["n_upheld"] = fmt_int(len(upheld))
    f["n_dismissed"] = fmt_int(len(dismissed))
    adjudicated = len(upheld) + len(dismissed)
    f["pct_upheld"] = pct(len(upheld), adjudicated)
    f["n_enf_notices"] = fmt_int(
        sum(1 for x in decisions if "enforcement notice issued" in x["outcome"]))
    f["n_prosecution_rec"] = fmt_int(
        sum(1 for x in decisions
            if "prosecution" in x["outcome"] and x["jurisdiction"] == "KE"))
    f["n_police_referral_mu"] = fmt_int(
        sum(1 for x in decisions
            if "police" in x["outcome"] and x["jurisdiction"] == "MU"))
    rtypes = Counter(x["respondent_type"] for x in decisions)
    f["n_resp_private"] = fmt_int(rtypes["private"])
    f["n_resp_public"] = fmt_int(rtypes["public"])
    f["n_resp_individual"] = fmt_int(rtypes["individual"])
    f["pct_resp_private"] = pct(rtypes["private"], len(decisions))

    # --------------------------------------------------------- violations
    viols = Counter()
    for x in decisions:
        for v in x.get("violations") or []:
            viols[v] += 1
    top_v = viols.most_common(8)
    f["top_violations_table"] = "\n".join(
        f"| {v} | {n} | {pct(n, len(decisions))} |" for v, n in top_v)
    f["n_viol_unlawful_processing"] = fmt_int(viols["unlawful processing"])
    f["n_viol_erasure"] = fmt_int(viols["failure to respond to erasure request"])
    f["n_viol_consent"] = fmt_int(viols["lack of consent"])
    f["n_viol_commercial"] = fmt_int(viols["commercial use of personal data"])
    f["n_viol_image"] = fmt_int(viols["unauthorised image/photo use"])
    f["n_viol_disclosure"] = fmt_int(viols["unlawful data disclosure"])
    f["n_viol_security"] = fmt_int(viols["failure to implement security measures"])
    f["n_viol_debt"] = fmt_int(viols["unlawful debt collection"])
    f["n_viol_registration"] = fmt_int(viols["failure to register with odpc"])
    f["n_viol_obstruction"] = fmt_int(viols["obstruction of data commissioner"])

    provs = Counter()
    for x in decisions:
        if x["jurisdiction"] != "KE":
            continue
        for p in x.get("provisions_cited") or []:
            # roll subsections into their parent section: s.56(1) -> s.56
            m = re.match(r"(s\.\d+)", p)
            if m:
                provs[m.group(1) + " DPA 2019"] += 1
    top_p = provs.most_common(6)
    f["top_provisions_table"] = "\n".join(
        f"| {p} | {n} |" for p, n in top_p)
    for sec in ("s.25", "s.26", "s.29", "s.30", "s.56", "s.65", "s.37", "s.32"):
        f[f"n_prov_{sec.replace('s.', 's')}"] = fmt_int(
            provs.get(f"{sec} DPA 2019", 0))

    # ------------------------------------------------------------- sectors
    sectors = Counter(x.get("sector") or "unclassified" for x in decisions)
    f["n_sector_general"] = fmt_int(sectors.pop("general", 0))
    top_s = sectors.most_common(8)
    f["top_sectors_table"] = "\n".join(
        f"| {s} | {n} |" for s, n in top_s)
    f["n_sector_lending"] = fmt_int(sectors.get("digital lending", 0))
    f["n_sector_banking"] = fmt_int(sectors.get("banking", 0))
    f["n_sector_tech"] = fmt_int(sectors.get("technology", 0))
    f["n_sector_education"] = fmt_int(sectors.get("education", 0))
    f["n_sector_health"] = fmt_int(sectors.get("healthcare", 0))

    # -------------------------------------------------------------- Kenya
    ke = [x for x in decisions if x["jurisdiction"] == "KE"]
    ke_upheld = [x for x in ke if "complaint upheld" in x["outcome"]]
    ke_dismissed = [x for x in ke if "complaint dismissed" in x["outcome"]]
    f["pct_ke_upheld"] = pct(len(ke_upheld), len(ke_upheld) + len(ke_dismissed))
    f["n_ke_h1"] = fmt_int(sum(1 for x in ke if H1_LO <= ym(x) <= H1_HI))
    f["n_ke_2025"] = fmt_int(sum(1 for x in ke if ym(x)[:4] == "2025"))
    f["n_ke_2024"] = fmt_int(sum(1 for x in ke if ym(x)[:4] == "2024"))
    f["n_ke_2023"] = fmt_int(sum(1 for x in ke if ym(x)[:4] == "2023"))

    # ------------------------------------------------------ status / appeals
    f["n_appealed"] = fmt_int(
        sum(1 for x in decisions if x["status"] == "appealed"))
    f["n_ongoing"] = fmt_int(
        sum(1 for x in decisions if x["status"] == "ongoing"))

    # ---------------------------------------------------- regional patterns
    regions = ["North Africa", "West Africa", "Central Africa",
               "East Africa", "Southern Africa"]
    rows = []
    for r in regions:
        cs = [c for c in landscape if c["region"] == r]
        rows.append("| {} | {} | {} | {} | {} |".format(
            r, len(cs),
            sum(1 for c in cs
                if (c.get("dp_law") or {}).get("status") == "in_force"),
            sum(1 for c in cs if c.get("enforcement_active")),
            sum(1 for c in cs if c.get("decisions_published"))))
    f["region_table"] = "\n".join(rows)

    # -------------------------------------------- top Kenya awards (linked)
    cslug = {"KE": "kenya", "NG": "nigeria", "ZA": "south-africa",
             "UG": "uganda"}

    def turl(x):
        reg = re.sub(r"[^a-z0-9]+", "-", x["regulator"].lower()).strip("-")
        return (f"https://research.lawlab.africa/tracker/"
                f"{cslug[x['jurisdiction']]}/{reg}/"
                f"{str(x['date_decided'])[:4]}/{x['slug']}/")

    top_awards = sorted((x for x in decisions if x.get("compensation_usd")),
                        key=lambda x: -x["compensation_usd"])[:5]
    rows = []
    for x in top_awards:
        yr = str(x["date_decided"])[:4]
        rows.append(
            "| [{}]({}) | {} | KES {:,} | {} |".format(
                x["respondent"], turl(x), yr,
                x["compensation_amount"], fmt_usd(x["compensation_usd"])))
    f["top_awards_table"] = "\n".join(rows)

    # ----------------------------------------------------- fx documentation
    f["kes_usd_2026"] = "129"   # year-average, tools/migrate.py KES_USD table
    return f


# ------------------------------------------------------------------ charts
def build_charts(decisions, landscape):
    import matplotlib
    matplotlib.use("Agg")
    import matplotlib.pyplot as plt
    if BRAND_STYLE.exists():
        plt.style.use(str(BRAND_STYLE))
    CHARTS.mkdir(exist_ok=True)
    cobalt, sky, gold, green, grey = (
        "#2168F2", "#7FB3FF", "#C9962D", "#59D483", "#8A93A6")

    def save(fig, name):
        for ext in ("svg", "png"):
            fig.savefig(CHARTS / f"{name}.{ext}", bbox_inches="tight",
                        facecolor="white")
        plt.close(fig)
        print(f"  chart: {name}.svg/.png")

    # 1 — quarterly volume, 2023-Q1 .. 2026-Q2
    q = Counter(quarter(x) for x in decisions if ym(x) >= "2023-01")
    labels = [f"{y}-Q{i}" for y in (2023, 2024, 2025, 2026)
              for i in (1, 2, 3, 4)][:14]
    vals = [q.get(l, 0) for l in labels]
    fig, ax = plt.subplots(figsize=(9, 4.2))
    colors = [sky if l >= "2026-Q2" else cobalt for l in labels]
    ax.bar(labels, vals, color=colors, width=0.62)
    for i, v in enumerate(vals):
        ax.text(i, v + 1, str(v), ha="center", fontsize=8.5, color="#5A5A5A")
    ax.set_title("Published enforcement actions per quarter, 2023–H1 2026")
    ax.set_ylabel("Actions published")
    ax.tick_params(axis="x", rotation=45)
    fig.text(0.12, -0.18,
             "Q2 2026 (lighter) reflects regulator publication lag, not a collapse in activity. "
             "One 2021 record (NITDA/SokoLoan) precedes the window.\n"
             "Source: decisions.json, Law Lab Africa Research, "
             "research.lawlab.africa/tracker.", fontsize=7.5, color="#8A93A6")
    save(fig, "fig1-quarterly-volume")

    # 2 — records by jurisdiction
    juris = Counter(x["country"] for x in decisions)
    pairs = juris.most_common()
    fig, ax = plt.subplots(figsize=(8, 3.2))
    names = [p[0] for p in pairs][::-1]
    counts = [p[1] for p in pairs][::-1]
    ax.barh(names, counts, color=cobalt, height=0.55)
    for i, v in enumerate(counts):
        ax.text(v + 3, i, str(v), va="center", fontsize=9, color="#5A5A5A")
    ax.set_title("Tracked enforcement records by jurisdiction (all years)")
    ax.set_xlim(0, max(counts) * 1.12)
    fig.text(0.12, -0.06,
             "Counts reflect dataset coverage depth, not total continental activity: "
             "Kenya's ODPC publishes individual determinations; most regulators do not.\n"
             "Source: decisions.json, Law Lab Africa Research.",
             fontsize=7.5, color="#8A93A6")
    save(fig, "fig2-jurisdictions")

    # 3 — the landscape funnel
    steps = [
        ("AU member states", 54),
        ("DP law in force",
         sum(1 for c in landscape
             if (c.get("dp_law") or {}).get("status") == "in_force")),
        ("Operational regulator",
         sum(1 for c in landscape
             if (c.get("regulator") or {}).get("operational"))),
        ("Actively enforcing",
         sum(1 for c in landscape if c.get("enforcement_active"))),
        ("Publishing decisions",
         sum(1 for c in landscape if c.get("decisions_published"))),
    ]
    fig, ax = plt.subplots(figsize=(8, 3.6))
    names = [s[0] for s in steps][::-1]
    vals = [s[1] for s in steps][::-1]
    shades = [cobalt, "#3F7DF4", "#6D9DF7", "#9CBEFA", "#CADEFc"][::-1]
    ax.barh(names, vals, color=shades, height=0.6)
    for i, v in enumerate(vals):
        ax.text(v + 0.7, i, str(v), va="center", fontsize=10, color="#1A1A1A")
    ax.set_title("The enforcement–publication funnel, 54 African Union states, mid-2026")
    ax.set_xlim(0, 60)
    fig.text(0.12, -0.05,
             "Source: landscape.json (54/54 countries verified with per-cell source "
             "URLs), Law Lab Africa Research, research.lawlab.africa/landscape.",
             fontsize=7.5, color="#8A93A6")
    save(fig, "fig3-funnel")

    # 4 — sectors (named sectors only)
    sectors = Counter(x.get("sector") for x in decisions
                      if x.get("sector") and x["sector"] != "general")
    pairs = sectors.most_common(10)
    fig, ax = plt.subplots(figsize=(8, 4.2))
    names = [p[0] for p in pairs][::-1]
    counts = [p[1] for p in pairs][::-1]
    ax.barh(names, counts, color=cobalt, height=0.6)
    for i, v in enumerate(counts):
        ax.text(v + 0.5, i, str(v), va="center", fontsize=9, color="#5A5A5A")
    ax.set_title("Enforcement actions by respondent sector (top 10 classified)")
    ax.set_xlim(0, max(counts) * 1.15)
    n_gen = sum(1 for x in decisions if x.get("sector") == "general")
    fig.text(0.12, -0.05,
             f"{n_gen} further records are classified 'general' (no dominant "
             "sector stated in the source document). Source: decisions.json.",
             fontsize=7.5, color="#8A93A6")
    save(fig, "fig4-sectors")

    # 5 — monetary orders, log scale, penalties vs compensation
    pens = sorted([x for x in decisions if x.get("penalty_usd")],
                  key=lambda x: x["penalty_usd"])
    comps = sorted([x for x in decisions if x.get("compensation_usd")],
                   key=lambda x: x["compensation_usd"])
    fig, ax = plt.subplots(figsize=(9, 4.4))
    ax.scatter([x["penalty_usd"] for x in pens], [1.0] * len(pens),
               s=60, color=cobalt, label=f"Administrative penalties (n={len(pens)})",
               zorder=3, alpha=0.85)
    ax.scatter([x["compensation_usd"] for x in comps], [0.0] * len(comps),
               s=60, color=gold, label=f"ODPC compensation orders (n={len(comps)})",
               zorder=3, alpha=0.75)
    pmed = statistics.median([x["penalty_usd"] for x in pens])
    cmed = statistics.median([x["compensation_usd"] for x in comps])
    ax.axvline(pmed, color=cobalt, lw=1, ls="--", alpha=0.6)
    ax.axvline(cmed, color=gold, lw=1, ls="--", alpha=0.6)
    ax.text(pmed * 1.15, 1.28, f"median USD {round(pmed):,}", fontsize=8.5,
            color=cobalt)
    ax.text(cmed * 1.15, -0.42, f"median USD {round(cmed):,}", fontsize=8.5,
            color="#8a6a1d")
    ax.set_xscale("log")
    ax.set_yticks([0, 1])
    ax.set_yticklabels(["Compensation\n(Kenya ODPC)", "Penalties\n(NG · ZA · UG)"])
    ax.set_ylim(-0.8, 1.8)
    ax.set_xlabel("USD, log scale (converted at decision-year average; original currency authoritative)")
    ax.set_title("Every disclosed monetary order in the dataset, 2021–H1 2026")
    ax.legend(loc="upper left", fontsize=8.5, frameon=False)
    fig.text(0.12, -0.04,
             "Rightmost point: FCCPC v Meta, USD 220m (Nigeria, 2024, consumer-protection statute). "
             "Medians, not means, describe the typical order.\n"
             "Source: decisions.json, Law Lab Africa Research.",
             fontsize=7.5, color="#8A93A6")
    save(fig, "fig5-monetary-orders")

    # 6 — top violation categories
    viols = Counter()
    for x in decisions:
        for v in x.get("violations") or []:
            viols[v] += 1
    pairs = viols.most_common(8)
    fig, ax = plt.subplots(figsize=(8.6, 4))
    names = [p[0] for p in pairs][::-1]
    counts = [p[1] for p in pairs][::-1]
    ax.barh(names, counts, color=cobalt, height=0.6)
    for i, v in enumerate(counts):
        ax.text(v + 2, i, str(v), va="center", fontsize=9, color="#5A5A5A")
    ax.set_title("Most-cited violation categories (a record can cite several)")
    ax.set_xlim(0, max(counts) * 1.12)
    fig.text(0.12, -0.05,
             "Kenya-weighted: 302 of 331 records are ODPC matters. "
             "Source: decisions.json, Law Lab Africa Research.",
             fontsize=7.5, color="#8A93A6")
    save(fig, "fig6-violations")


# ------------------------------------------------------------------ render
def render(figures):
    if not TEMPLATE.exists():
        print("template missing; skipping render")
        return
    text = TEMPLATE.read_text()
    unresolved = []

    def sub(m):
        key = m.group(1)
        if key in figures:
            return str(figures[key])
        unresolved.append(key)
        return m.group(0)

    out = re.sub(r"\{\{([a-z0-9_]+)\}\}", sub, text)
    if unresolved:
        sys.exit(f"FATAL: unresolved tokens: {sorted(set(unresolved))}")
    REPORT.write_text(out)
    words = len(re.sub(r"[|#*\-`]", " ", out).split())
    print(f"report rendered: {REPORT.name} ({words:,} words incl. tables)")


def main():
    decisions, landscape = load()
    figures = compute(decisions, landscape)
    (HERE / "figures.json").write_text(
        json.dumps(figures, indent=1, ensure_ascii=False) + "\n")
    print(f"figures.json: {len(figures)} figures")
    if "--charts" in sys.argv:
        build_charts(decisions, landscape)
    render(figures)


if __name__ == "__main__":
    main()