#!/usr/bin/env python3
"""test_challenges.py -- the LintQ challenge examples as real unit tests.

Each (example, detector) pair from ``challenges.json`` becomes a ``unittest``
test case, so the examples double as regression tests for the detectors.  The
shared acceptance logic in ``challenge_lib.py`` is reused verbatim, which keeps
the unit tests and the ``verify_challenges.py`` CI gate in lock-step.

Tests only assert *explicit* expectations: a detector that is expected to FIRE
must emit a warning, a detector expected to stay SILENT on a negative example
must not, and a detector with a documented limitation (FALSE_POS) must not
fire.  ``N/A`` rows (detectors structurally out of scope for an example) are
skipped -- they are reported by the CI gate, not asserted here.

Run:
    python3 -m unittest examples.test_challenges
"""

from __future__ import annotations

import os
import sys
import tempfile
import unittest

HERE = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, HERE)
import challenge_lib as lib  # noqa: E402


class ChallengeTestBase(unittest.TestCase):
    """Shared harness: evaluates every example once and lets subclasses
    query the per-(example, detector) results."""

    @classmethod
    def setUpClass(cls):
        import shutil as _shutil
        if not _shutil.which("souffle"):
            raise unittest.SkipTest("souffle not on PATH")
        cls.data = lib.load_challenges()
        cls._tmp = tempfile.mkdtemp(prefix="lintq_ct_")
        cls.results = []
        for ex in cls.data["examples"]:
            cls.results.extend(lib.evaluate_example(ex, cls._tmp))

    @classmethod
    def tearDownClass(cls):
        import shutil
        shutil.rmtree(cls._tmp, ignore_errors=True)

    def _result_for(self, ex_id, detector):
        for r in self.results:
            if r.ex_id == ex_id and r.detector == detector:
                return r
        self.fail(f"no result for {ex_id}/{detector}")


def _make_test(ex_id: str, det: str, expected: str, skip: bool):
    """Build a test method. ``skip`` means the detector is N/A for this
    example and is intentionally not asserted here."""
    def test(self):
        r = self._result_for(ex_id, det)
        if skip:
            self.skipTest(f"detector {det} is N/A for {ex_id}")
        self.assertFalse(
            r.failed,
            f"{ex_id}/{det}: expected {expected} but got verdict "
            f"{r.verdict} (fired={r.did_fire})")
    test.__name__ = f"test_{ex_id}__{det}"
    test.__doc__ = (f"{ex_id}: {det} expected {expected}"
                    f" -> {'SKIPPED (N/A)' if skip else 'asserted'}")
    return test


def _build():
    """Generate one test method per (example, detector) pair and attach them
    to ``ChallengeTests`` so they appear as individual unittest cases."""
    data = lib.load_challenges()
    attrs = {}
    for ex in data["examples"]:
        for det, expected in ex["ground_truth"].items():
            skip = (expected == "N/A")
            attrs[f"test_{ex['id']}__{det}"] = _make_test(
                ex["id"], det, expected, skip)
    return attrs


# Dynamically populate ChallengeTests with a method per (example, detector).
ChallengeTests = type(
    "ChallengeTests", (ChallengeTestBase,), _build())


if __name__ == "__main__":
    unittest.main()