#!/usr/bin/env python3
"""eval_report.py -- build a 10-example evaluation set, run the LintQ
detectors on it, capture the raw output, and render a self-contained HTML
report with per-detector precision/recall.

This is a *subset* evaluation of the full 17-example challenge corpus
(challenges.json). It picks 10 representative examples that exercise all
four bug classes (OpAfterMeas, DoubleMeas, MissingReset, GhostCompose),
include positive (FIRE) and negative (SILENT) cases, a documented
reset-blind limitation (FALSE_POS), and a cross-detector interaction.

For each example the script runs BOTH real LintQ engines:

  * Engine A -- ``engine/lintq.dl`` over the extractor EDB schema
                (detectors: OpAfterMeas, DoubleMeas, GhostCompose).
  * Engine B -- ``rules/lintq.dl`` over the compact ``Op/ActsOn/Succ``
                schema (detectors: OpAfterMeas, DoubleMeas, MissingReset).

and captures the raw ``Warn*.csv`` content each engine emits, so the
report shows exactly what the detectors produced.

Metrics
-------
A detector is evaluated only on the rows where it is *in scope*
(its owning engine implements it AND the ground-truth label is not
``N/A``).  ``N/A`` rows are out of scope and excluded from the
denominator entirely.

The ground-truth label says what the detector should do:

  FIRE       bug class genuinely present  -> detector MUST emit a warning
  SILENT     bug class intentionally absent -> detector MUST stay silent
  FALSE_POS  documented engine limitation  -> a reset-blind detector is
             EXPECTED to fire here; the fire is the known, non-buggy
             manifestation of the limitation, so it is counted as a
             *correct* positive (not penalised).

From the (expected, actual) pairs we derive the standard confusion
matrix using "fire" as the positive class:

  TP  = fired   & expected-fire
  FP  = fired   & expected-silent          (genuine false alarm)
  FN  = silent  & expected-fire            (missed bug)
  TN  = silent  & expected-silent

  precision = TP / (TP + FP)
  recall    = TP / (TP + FN)

``limitations`` = number of FALSE_POS rows (fires counted as correct
positives but reported separately for honesty).

Usage:
    python3 examples/eval_report.py
        -> writes examples/eval/eval_report.html and eval/eval_results.json
"""

from __future__ import annotations

import html
import json
import os
import subprocess
import tempfile

HERE = os.path.dirname(os.path.abspath(__file__))
REPO = os.path.dirname(HERE)

# Import the shared harness for running/evaluating (single source of truth).
import sys
sys.path.insert(0, HERE)
import challenge_lib as lib  # noqa: E402

# The 10 examples chosen for this eval subset (ids from challenges.json).
EVAL_IDS = [
    "ex01_op_after_meas_basic",        # OpAfterMeas FIRE (basic)
    "ex02_op_after_meas_reset_safe",   # OpAfterMeas FALSE_POS for Engine A
    "basic_reset_initialised",          # OpAfterMeas FIRE via 2-qubit cx (ex03 file)
    "ex06_double_meas_basic",          # DoubleMeas FIRE (basic)
    "ex07_double_meas_reset_between",  # DoubleMeas SILENT negative
    "ex10_double_meas_2qubitgate",     # DoubleMeas SILENT + OpAfterMeas FIRE
    "ex11_missing_reset_basic",        # MissingReset FIRE (basic)
    "ex14_missing_reset_2qubit_gate",  # MissingReset FIRE via 2-qubit cx
    "ex15_ghost_compose_basic",        # GhostCompose FIRE (basic)
    "ex16_ghost_compose_assigned",     # GhostCompose SILENT negative
]

ALL_WARNINGS = lib.ALL_WARNINGS
DETECTOR_TO_WARN = lib.DETECTOR_TO_WARN
WARN_DISPLAY = {
    "WarnOpAfterMeas": "OpAfterMeas",
    "WarnDoubleMeas": "DoubleMeas",
    "WarnGhostCompose": "GhostCompose",
    "WarnMissingReset": "MissingReset",
}


def run_engine_raw(dl: str, facts_dir: str, out_dir: str) -> dict:
    """Run Souffle and return {warn_relation: raw_csv_string} for every
    warning relation (empty string if the relation produced no rows)."""
    os.makedirs(out_dir, exist_ok=True)
    subprocess.run(
        ["souffle", "-F", facts_dir, "-D", out_dir, dl],
        check=True, capture_output=True, text=True)
    raw: dict[str, str] = {}
    for name in ALL_WARNINGS:
        path = os.path.join(out_dir, name + ".csv")
        if os.path.exists(path):
            with open(path) as fh:
                raw[name] = fh.read().strip("\n")
        else:
            raw[name] = ""
    return raw


def expected_fire(label: str) -> bool:
    """What the detector *should* do for a given ground-truth label."""
    if label == "FIRE":
        return True
    if label == "SILENT":
        return False
    if label == "FALSE_POS":
        # documented limitation: the engine firing is the expected behaviour
        return True
    # N/A -> out of scope, handled by caller
    return False


def evaluate_one(ex: dict, tmp_root: str) -> dict:
    """Run one example through both engines; return a structured record
    with raw output, verdicts, and per-detector confusion contributions."""
    sys.path.insert(0, os.path.join(REPO, "extractor"))
    import ast_to_facts  # noqa: E402

    ex_id = ex["id"]
    labels = ex["ground_truth"]
    src = os.path.join(HERE, ex["file"])
    with open(src) as fh:
        source = fh.read()

    # Engine A
    a_facts = os.path.join(tmp_root, ex_id, "A")
    ext = ast_to_facts.QiskitFactExtractor()
    ext.run(source, a_facts)
    a_out = os.path.join(tmp_root, ex_id, "A_out")
    a_raw = run_engine_raw(lib.ENGINE_A_DL, a_facts, a_out)

    # Engine B (compact facts synthesised from the same extractor)
    b_facts = os.path.join(tmp_root, ex_id, "B")
    os.makedirs(b_facts, exist_ok=True)
    lib.write_compact_facts(ext, b_facts)
    b_out = os.path.join(tmp_root, ex_id, "B_out")
    b_raw = run_engine_raw(lib.ENGINE_B_DL, b_facts, b_out)

    # Build per-detector rows
    detectors = []
    for det, label in labels.items():
        warn = DETECTOR_TO_WARN[det]
        engine = det[0]
        raw_all = a_raw if engine == "A" else b_raw
        did_fire = bool(raw_all.get(warn, "").strip())
        capable = lib.detector_capable(det)
        if not capable:
            verdict = "N/A"
        else:
            verdict = lib.classify_verdict(label, did_fire)
        detectors.append({
            "detector": det,
            "bug": lib.DETECTOR_BUG[det][1],
            "engine": engine,
            "expected": label,
            "fired": did_fire,
            "capable": capable,
            "verdict": verdict,
            "raw_csv": raw_all.get(warn, "") if capable else "",
        })

    return {
        "id": ex_id,
        "file": ex["file"],
        "title": ex["title"],
        "category": ex.get("category", ""),
        "bug_class": ex.get("bug_class", ""),
        "description": ex.get("description", ""),
        "notes": ex.get("notes", ""),
        "source": source,
        "a_raw": a_raw,
        "b_raw": b_raw,
        "detectors": detectors,
    }


def compute_metrics(records: list[dict]) -> dict:
    """Aggregate per-detector confusion matrix + precision/recall over all
    in-scope (example, detector) rows."""
    # per-detector accumulators
    agg = {}
    for rec in records:
        for d in rec["detectors"]:
            if not d["capable"]:
                continue
            det = d["detector"]
            a = agg.setdefault(det, {
                "detector": det, "bug": d["bug"], "engine": d["engine"],
                "tp": 0, "fp": 0, "fn": 0, "tn": 0, "na": 0, "limit": 0,
                "rows": 0,
            })
            a["rows"] += 1
            if d["expected"] == "N/A":
                a["na"] += 1
                continue
            exp_fire = expected_fire(d["expected"])
            if d["expected"] == "FALSE_POS":
                a["limit"] += 1
            if d["fired"] and exp_fire:
                a["tp"] += 1
            elif d["fired"] and not exp_fire:
                a["fp"] += 1
            elif (not d["fired"]) and exp_fire:
                a["fn"] += 1
            else:
                a["tn"] += 1

    for det, a in agg.items():
        tp, fp, fn = a["tp"], a["fp"], a["fn"]
        a["precision"] = (tp / (tp + fp)) if (tp + fp) else None
        a["recall"] = (tp / (tp + fn)) if (tp + fn) else None
        a["f1"] = (
            (2 * tp / (2 * tp + fp + fn))
            if (2 * tp + fp + fn) else None
        )
        a["accuracy"] = (
            (tp + a["tn"]) / (tp + a["tn"] + fp + fn)
            if (tp + a["tn"] + fp + fn) else None
        )
    return agg


def esc(s: str) -> str:
    return html.escape(s)


def raw_block(raw: str) -> str:
    if not raw.strip():
        return '<pre class="empty">(no warnings emitted)</pre>'
    return f'<pre class="csv">{esc(raw)}</pre>'


def render_html(records: list[dict], metrics: dict, meta: dict) -> str:
    # ---- per-detector metrics table ----
    metric_rows = []
    for det in sorted(metrics):
        a = metrics[det]
        def fmt(x):
            return "—" if x is None else f"{x*100:.1f}%"
        metric_rows.append(
            "<tr>"
            f"<td class='mono'>{esc(det)}</td>"
            f"<td>{esc(a['bug'])}</td>"
            f"<td>{esc(a['engine'])}</td>"
            f"<td>{a['tp']}</td><td>{a['fp']}</td>"
            f"<td>{a['fn']}</td><td>{a['tn']}</td>"
            f"<td class='hl'>{fmt(a['precision'])}</td>"
            f"<td class='hl'>{fmt(a['recall'])}</td>"
            f"<td>{fmt(a['f1'])}</td>"
            f"<td>{a['na']}</td><td>{a['limit']}</td>"
            "</tr>"
        )
    metric_table = (
        "<table class='metrics'>"
        "<thead><tr>"
        "<th>Detector</th><th>Bug class</th><th>Engine</th>"
        "<th>TP</th><th>FP</th><th>FN</th><th>TN</th>"
        "<th>Precision</th><th>Recall</th><th>F1</th>"
        "<th>N/A excl.</th><th>Limitation</th>"
        "</tr></thead><tbody>"
        + "".join(metric_rows)
        + "</tbody></table>"
    )

    # ---- per-example detail ----
    example_blocks = []
    for rec in records:
        det_rows = []
        for d in rec["detectors"]:
            cls = "na" if d["verdict"] == "N/A" else (
                "ok" if d["verdict"] in ("FIRE", "SILENT", "SILENT(limitation)")
                else "bad" if d["verdict"] in ("FALSE_NEG", "FALSE_POS")
                else "lim")
            det_rows.append(
                "<tr class='" + cls + "'>"
                f"<td class='mono'>{esc(d['detector'])}</td>"
                f"<td>{esc(d['expected'])}</td>"
                f"<td>{'FIRE' if d['fired'] else 'silent'}</td>"
                f"<td><b>{esc(d['verdict'])}</b></td>"
                f"<td>{raw_block(d['raw_csv'])}</td>"
                "</tr>"
            )
        det_table = (
            "<table class='det'>"
            "<thead><tr><th>Detector</th><th>Expected</th>"
            "<th>Actual</th><th>Verdict</th><th>Raw detector output</th>"
            "</tr></thead><tbody>"
            + "".join(det_rows)
            + "</tbody></table>"
        )
        example_blocks.append(
            f"<section class='ex' id='{esc(rec['id'])}'>"
            f"<h3>{esc(rec['title'])} "
            f"<span class='tag'>{esc(rec['bug_class'])}</span></h3>"
            f"<p class='desc'>{esc(rec['description'])}</p>"
            f"<details><summary>Qiskit source ({esc(rec['file'])})</summary>"
            f"<pre class='src'>{esc(rec['source'])}</pre></details>"
            + det_table
            + (f"<p class='notes'><i>Note:</i> {esc(rec['notes'])}</p>"
               if rec.get("notes") else "")
            + "</section>"
        )
    example_html = "\n".join(example_blocks)

    # ---- bug-class coverage summary ----
    class_hits = {}
    for rec in records:
        for d in rec["detectors"]:
            if d["capable"] and d["expected"] in ("FIRE", "SILENT", "FALSE_POS"):
                class_hits.setdefault(d["bug"], set()).add(rec["id"])
    cov_rows = "".join(
        f"<tr><td>{esc(b)}</td>"
        f"<td>{', '.join(sorted(class_hits.get(b, [])) or ['—'])}</td></tr>"
        for b in ("OpAfterMeas", "DoubleMeas", "MissingReset", "GhostCompose")
    )
    coverage = (
        "<table class='cov'><thead><tr><th>Bug class</th>"
        "<th>Eval examples covering it</th></tr></thead><tbody>"
        + cov_rows + "</tbody></table>"
    )

    total_tp = sum(a["tp"] for a in metrics.values())
    total_fp = sum(a["fp"] for a in metrics.values())
    total_fn = sum(a["fn"] for a in metrics.values())
    overall_p = (total_tp / (total_tp + total_fp)) if (total_tp + total_fp) else None
    overall_r = (total_tp / (total_tp + total_fn)) if (total_tp + total_fn) else None

    return f"""<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>LintQ Detector Eval Report</title>
<style>
:root {{
  --bg:#0d1117; --panel:#161b22; --border:#30363d; --fg:#e6edf3;
  --muted:#8b949e; --accent:#58a6ff; --ok:#3fb950; --bad:#f85149;
  --lim:#d29922; --na:#6e7681;
}}
* {{ box-sizing:border-box; }}
body {{ background:var(--bg); color:var(--fg); font-family:-apple-system,
  Segoe UI, Roboto, Helvetica, Arial, sans-serif; margin:0; padding:0 0 4rem;
  line-height:1.5; }}
header {{ padding:2rem 2rem 1rem; border-bottom:1px solid var(--border);
  background:linear-gradient(180deg,#161b22,#0d1117); }}
header h1 {{ margin:0 0 .3rem; font-size:1.7rem; }}
header p {{ margin:.2rem 0; color:var(--muted); }}
main {{ max-width:1100px; margin:0 auto; padding:0 1.5rem; }}
h2 {{ margin-top:2.5rem; border-left:4px solid var(--accent);
  padding-left:.6rem; }}
section.ex, table {{ background:var(--panel); border:1px solid var(--border);
  border-radius:8px; }}
section.ex {{ padding:1rem 1.2rem; margin:1.2rem 0; }}
section.ex h3 {{ margin:.2rem 0 .4rem; }}
.tag {{ font-size:.7rem; background:#21262d; color:var(--accent);
  border:1px solid var(--border); border-radius:10px; padding:.1rem .5rem;
  vertical-align:middle; }}
.desc {{ color:var(--muted); margin:.2rem 0 .8rem; }}
table {{ border-collapse:collapse; width:100%; margin:1rem 0;
  font-size:.9rem; overflow:hidden; }}
th, td {{ text-align:left; padding:.5rem .6rem; border-bottom:1px solid var(--border);
  vertical-align:top; }}
th {{ background:#1c2330; color:var(--fg); font-weight:600; }}
td.hl {{ font-weight:700; color:var(--accent); }}
.mono {{ font-family:ui-monospace,SFMono-Regular,Menlo,monospace; }}
pre {{ background:#010409; border:1px solid var(--border); border-radius:6px;
  padding:.6rem; overflow:auto; font-size:.82rem; }}
pre.empty {{ color:var(--muted); font-style:italic; }}
pre.csv, pre.src {{ margin:0; }}
pre.src {{ color:#c9d1d9; }}
tr.ok td:nth-child(4) {{ color:var(--ok); }}
tr.bad td:nth-child(4) {{ color:var(--bad); }}
tr.lim td:nth-child(4) {{ color:var(--lim); }}
tr.na td:nth-child(4) {{ color:var(--na); }}
.legend span {{ display:inline-block; margin-right:1rem; font-size:.85rem; }}
.dot {{ display:inline-block; width:.7rem; height:.7rem; border-radius:50%;
  margin-right:.3rem; vertical-align:middle; }}
.d-ok {{ background:var(--ok); }} .d-bad {{ background:var(--bad); }}
.d-lim {{ background:var(--lim); }} .d-na {{ background:var(--na); }}
details {{ margin:.6rem 0; }}
summary {{ cursor:pointer; color:var(--accent); }}
.notes {{ color:var(--muted); font-size:.85rem; }}
.kpi {{ display:flex; gap:1rem; flex-wrap:wrap; margin:1rem 0; }}
.kpi div {{ background:var(--panel); border:1px solid var(--border);
  border-radius:8px; padding:.8rem 1.2rem; }}
.kpi b {{ font-size:1.4rem; color:var(--accent); display:block; }}
code {{ background:#161b22; padding:.1rem .3rem; border-radius:4px; }}
footer {{ max-width:1100px; margin:2rem auto 0; padding:0 1.5rem;
  color:var(--muted); font-size:.8rem; }}
</style>
</head>
<body>
<header>
  <h1>LintQ Detector Evaluation Report</h1>
  <p>{meta['n_examples']} representative examples &middot; {meta['n_detectors']}
     detectors &middot; both real engines (A + B) under Souffl&eacute; 2.5</p>
  <p>Generated {esc(meta['generated'])} &middot; subset of the 17-example
     challenge corpus (<code>challenges.json</code>)</p>
</header>
<main>

<h2>How this report is presented</h2>
<p>Each example below shows, for every detector, four things: the
<b>expected</b> ground-truth label, the <b>actual</b> detector behaviour
(fired / silent), the derived <b>verdict</b>, and the <b>raw output</b> the
engine actually emitted (the literal <code>Warn*.csv</code> rows). The raw
output is the truth &mdash; it is what the detector printed, not a
re-description of it.</p>
<div class="legend">
  <span><span class="dot d-ok"></span>correct (FIRE / SILENT as expected)</span>
  <span><span class="dot d-bad"></span>regression (FALSE_NEG / FALSE_POS)</span>
  <span><span class="dot d-lim"></span>documented limitation</span>
  <span><span class="dot d-na"></span>out of scope (N/A)</span>
</div>

<p><b>Ground-truth label vocabulary.</b>
<code>FIRE</code> &mdash; the bug class is genuinely present, the detector must
warn. <code>SILENT</code> &mdash; the bug is intentionally absent, the detector
must stay silent (negative example).
<code>FALSE_POS</code> &mdash; a documented engine limitation: a reset-blind
detector is <i>expected</i> to fire here, so the fire is the known, non-buggy
manifestation and is counted as a correct positive (reported separately as a
"limitation"). <code>N/A</code> &mdash; the detector is structurally unable to
observe this bug (e.g. Engine A has no MissingReset rule); these rows are
excluded from the metrics denominator.</p>

<h2>Precision &amp; recall &mdash; methodology</h2>
<p>A detector is scored only on rows where it is in scope (it is implemented by
its engine and the label is not <code>N/A</code>). "Fire" is the positive
class. For each in-scope row we compare the expected behaviour to the actual
firing:</p>
<ul>
  <li><b>TP</b> &mdash; fired &amp; expected-fire</li>
  <li><b>FP</b> &mdash; fired &amp; expected-silent (a genuine false alarm)</li>
  <li><b>FN</b> &mdash; silent &amp; expected-fire (a missed bug)</li>
  <li><b>TN</b> &mdash; silent &amp; expected-silent</li>
</ul>
<p><code>FALSE_POS</code> rows are treated as expected-fire, so their fire
counts as <b>TP</b> (not FP) &mdash; this measures each detector against its
<i>documented as-built contract</i>, exactly as the CI gate does. They are also
tallied in the "Limitation" column so the limitation is never hidden.
Precision = TP / (TP + FP); Recall = TP / (TP + FN).</p>

<div class="kpi">
  <div><b>{total_tp}</b>total TP</div>
  <div><b>{total_fp}</b>total FP</div>
  <div><b>{total_fn}</b>total FN</div>
  <div><b>{fmt(overall_p)}</b>overall precision</div>
  <div><b>{fmt(overall_r)}</b>overall recall</div>
</div>

<h3>Per-detector metrics</h3>
{metric_table}

<h2>Bug-class coverage of the eval set</h2>
{coverage}

<h2>Per-example results (raw detector output)</h2>
{example_html}

</main>
<footer>
LintQ Souffl&eacute; reimplementation &middot; eval subset generated by
<code>examples/eval_report.py</code> &middot; both engines run live under
Souffl&eacute; 2.5, raw CSV captured directly.
</footer>
</body>
</html>"""


def fmt(x):
    return "—" if x is None else f"{x*100:.1f}%"


def main() -> int:
    if not shutil_which("souffle"):
        print("souffle not on PATH", file=sys.stderr)
        return 2
    data = lib.load_challenges()
    by_id = {e["id"]: e for e in data["examples"]}
    chosen = [by_id[i] for i in EVAL_IDS if i in by_id]

    tmp_root = tempfile.mkdtemp(prefix="lintq_eval_")
    records = [evaluate_one(ex, tmp_root) for ex in chosen]
    metrics = compute_metrics(records)

    from datetime import datetime, timezone
    meta = {
        "n_examples": len(records),
        "n_detectors": len(metrics),
        "generated": datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"),
        "eval_ids": EVAL_IDS,
        "souffle": "2.5",
    }

    # Persist raw structured results alongside the HTML (transparency).
    out_dir = os.path.join(HERE, "eval")
    os.makedirs(out_dir, exist_ok=True)
    with open(os.path.join(out_dir, "eval_results.json"), "w") as fh:
        json.dump({"meta": meta, "records": records,
                   "metrics": metrics}, fh, indent=2)

    html_doc = render_html(records, metrics, meta)
    html_path = os.path.join(out_dir, "eval_report.html")
    with open(html_path, "w") as fh:
        fh.write(html_doc)

    # Console summary
    print(f"Evaluated {len(records)} examples, {len(metrics)} detectors")
    for det in sorted(metrics):
        a = metrics[det]
        print(f"  {det:14s} P={fmt(a['precision']):>5s} "
              f"R={fmt(a['recall']):>5s}  "
              f"(TP={a['tp']} FP={a['fp']} FN={a['fn']} TN={a['tn']} "
              f"N/A={a['na']} lim={a['limit']})")
    print(f"Report: {html_path}")
    return 0


def shutil_which(name):
    import shutil
    return shutil.which(name)


if __name__ == "__main__":
    sys.exit(main())