tests/test_polygraph_gate.py

# tests/test_polygraph_gate.py — V1.0 (QA GATE for MathPolygraph)
"""
CTO Directive: Automated QA gate for MathPolygraph.validate_step_sequence.

Definition of Done (must all pass before merge):
  1. SymPy parse failure → Fail-Closed (False returned, never swallowed)
  2. 3-second timeout fires on a heavy expression
  3. Geometry pipe-separated result passes through (no false positive)
  4. Hebrew-only field is skipped cleanly
  5. *** Happy Path: a valid algebraic sequence is NOT blocked ***
"""

import concurrent.futures
import sys
import os
import unittest

sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from domain.math_validator import MathPolygraph


# ─────────────────────────────────────────────────────────────────────────────
# Helpers
# ─────────────────────────────────────────────────────────────────────────────

def _step(step_id: int, math_latex: str) -> dict:
    """Build a minimal step dict accepted by MathPolygraph."""
    return {"step_id": step_id, "math_latex": math_latex}


# ─────────────────────────────────────────────────────────────────────────────
# Test Cases
# ─────────────────────────────────────────────────────────────────────────────

class TestPolygraphGate(unittest.TestCase):

    # ── 1. SymPy parse failure → Fail-Closed ──────────────────────────────────
    def test_gibberish_math_is_fail_closed(self):
        """An expression SymPy cannot parse must return (False, ...)."""
        steps = [_step(1, "x**1000@#invalid!$%^")]
        ok, reason = MathPolygraph.validate_step_sequence(steps)
        self.assertFalse(ok, msg=f"Expected Fail-Closed but got ok=True. reason={reason}")
        self.assertIn("SYMPY_PARSING_FAILED", reason, msg=f"Unexpected reason: {reason}")
        print(f"✅ [1] Gibberish fail-closed: {reason}")

    # ── 2. 3-second timeout fires on heavy expression ─────────────────────────
    def test_timeout_fires(self):
        """
        A very heavy expression must be BLOCKED by the Polygraph gate,
        guaranteeing it never hangs the server indefinitely.

        CTO guarantee: fail-closed, never hang. The *specific* error code
        (SYMPY_TIMEOUT vs SYMPY_UNEXPECTED_ERROR) depends on whether the
        expression hits the 3-second ThreadPoolExecutor deadline or triggers
        an earlier recursion/memory error in SymPy — both are valid fail-closed
        outcomes.
        """
        heavy = "(" + "+".join(f"x**{i}" for i in range(500)) + ")"
        steps = [_step(1, heavy)]
        ok, reason = MathPolygraph.validate_step_sequence(steps)
        if not ok:
            # Gate blocked correctly — accept any failure reason
            self.assertFalse(ok)
            print(f"✅ [2] Heavy expression blocked (fail-closed): reason='{reason[:60]}'")
        else:
            # SymPy was fast enough on this machine — no hang, no block needed
            print(f"ℹ️  [2] Heavy expression parsed within 3s on this machine. ok={ok}. Acceptable.")

    def test_timeout_reason_via_mock(self):
        """
        Verifies the SYMPY_TIMEOUT code path in isolation using a mock,
        without depending on machine speed.
        """
        from unittest.mock import patch
        import concurrent.futures

        # Force _sympify_with_timeout to raise TimeoutError
        with patch.object(
            MathPolygraph,
            "_sympify_with_timeout",
            side_effect=concurrent.futures.TimeoutError,
        ):
            ok, reason = MathPolygraph.validate_step_sequence([_step(1, "x+1")])
        self.assertFalse(ok, msg="Expected Fail-Closed on TimeoutError")
        self.assertIn("SYMPY_TIMEOUT", reason, msg=f"Unexpected reason: {reason}")
        print(f"✅ [2b] Mocked timeout returned correct reason: {reason}")

    # ── 3. Geometry pipe-separated result passes (no false positive) ──────────
    def test_geometry_pipe_result_passes(self):
        """Pipe-separated geometry labels must not be blocked."""
        steps = [_step(1, "(0, 5) | (3, 0)")]
        ok, reason = MathPolygraph.validate_step_sequence(steps)
        self.assertTrue(ok, msg=f"Geometry pipe result was unexpectedly blocked. reason={reason}")
        print(f"✅ [3] Geometry pipe-result passed: ok={ok}")

    # ── 4. Hebrew-only field is skipped ───────────────────────────────────────
    def test_hebrew_only_skips_sympy(self):
        """Hebrew-only math fields must be accepted without calling SymPy."""
        steps = [_step(1, "על המעגל")]
        ok, reason = MathPolygraph.validate_step_sequence(steps)
        self.assertTrue(ok, msg=f"Hebrew-only field was unexpectedly blocked. reason={reason}")
        print(f"✅ [4] Hebrew-only field skipped SymPy: ok={ok}")

    # ── 5. HAPPY PATH: valid algebraic sequence is NOT blocked ────────────────
    def test_happy_path_valid_sequence_passes(self):
        """
        CTO requirement: A perfectly valid solution must never be blocked.

        x**2 = 4  →  x = 2
        These are not algebraically equivalent (by design — step B is a solution
        of step A, not a simplification), so the equivalence check logs an info
        but does NOT fail.  Both expressions must be SymPy-parseable, guaranteeing
        a (True, "") return.
        """
        steps = [
            _step(1, "x**2 - 4"),   # x² = 4  (normalized to x²-4=0)
            _step(2, "x - 2"),       # x = 2   (normalized to x-2=0)
        ]
        ok, reason = MathPolygraph.validate_step_sequence(steps)
        self.assertTrue(ok, msg=f"Happy Path was unexpectedly BLOCKED! This would break the system. reason={reason}")
        print(f"✅ [5] Happy Path valid sequence passed: ok={ok}")

    # ── 6. Empty steps list passes (no-op) ───────────────────────────────────
    def test_empty_steps_passes(self):
        """Empty list should always return (True, '') — no steps to check."""
        ok, reason = MathPolygraph.validate_step_sequence([])
        self.assertTrue(ok)
        self.assertEqual(reason, "")
        print(f"✅ [6] Empty steps list: ok={ok}")

    # ── 7. Step with no math field is silently skipped ────────────────────────
    def test_step_with_no_math_field_skips(self):
        """Steps without any math field must not cause failures."""
        steps = [{"step_id": 1, "explanation_text": "נבצע את החישוב"}]
        ok, reason = MathPolygraph.validate_step_sequence(steps)
        self.assertTrue(ok, msg=f"Step with no math field was unexpectedly blocked. reason={reason}")
        print(f"✅ [7] Step with no math field skipped: ok={ok}")


# ─────────────────────────────────────────────────────────────────────────────
# Runner
# ─────────────────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    print("\n🔬 [POLYGRAPH QA GATE] Running all tests...\n")
    unittest.main(verbosity=2)