Support @expectedFailure in unittest

python-stdlib/unittest-discover/unittest/__main__.py

@@ -146,8 +146,8 @@ def discover_main():
         # Ensure an appropriate output is printed if no tests are found.
         runner.run(TestSuite())
 
-    # Terminate with non zero return code in case of failures.
-    sys.exit(result.failuresNum + result.errorsNum)
+    # Non-zero exit on failures; an unexpected success counts as a failure too.
+    sys.exit(result.failuresNum + result.errorsNum + result.unexpectedSuccessesNum)
 
 
 discover_main()

python-stdlib/unittest/tests/test_expectedfailure.py

@@ -0,0 +1,84 @@
+# Tests for @unittest.expectedFailure: a decorated test that fails is an expected
+# failure (run stays OK); one that passes is an unexpected success (run fails).
+# Each case runs through its own TestResult so it can't fail the outer run.
+import unittest
+
+
+def _run(test_cls):
+    result = unittest.TestResult()
+    suite = unittest.TestSuite()
+    suite.addTest(test_cls)
+    suite.run(result)
+    return result
+
+
+class TestExpectedFailure(unittest.TestCase):
+    def test_expected_failure_is_not_a_failure(self):
+        class Inner(unittest.TestCase):
+            @unittest.expectedFailure
+            def test_x(self):
+                self.assertEqual(1, 0)  # fails as expected
+
+        r = _run(Inner)
+        self.assertEqual(r.testsRun, 1)
+        self.assertEqual(r.expectedFailuresNum, 1)
+        self.assertEqual(r.failuresNum, 0)
+        self.assertEqual(r.errorsNum, 0)
+        self.assertTrue(r.wasSuccessful())
+
+    def test_unexpected_success_is_tracked_separately(self):
+        class Inner(unittest.TestCase):
+            @unittest.expectedFailure
+            def test_x(self):
+                self.assertEqual(1, 1)  # passes -> unexpected success
+
+        r = _run(Inner)
+        self.assertEqual(r.testsRun, 1)
+        self.assertEqual(r.unexpectedSuccessesNum, 1)
+        # Not a failure or error, but still makes the run unsuccessful.
+        self.assertEqual(r.failuresNum, 0)
+        self.assertEqual(r.errorsNum, 0)
+        self.assertFalse(r.wasSuccessful())
+
+    def test_regular_failure_still_counts_as_failure(self):
+        class Inner(unittest.TestCase):
+            def test_x(self):
+                self.assertEqual(1, 0)
+
+        r = _run(Inner)
+        self.assertEqual(r.failuresNum, 1)
+        self.assertEqual(r.expectedFailuresNum, 0)
+        self.assertEqual(r.unexpectedSuccessesNum, 0)
+        self.assertFalse(r.wasSuccessful())
+
+    def test_error_is_unaffected(self):
+        class Inner(unittest.TestCase):
+            def test_x(self):
+                raise ValueError("boom")
+
+        r = _run(Inner)
+        self.assertEqual(r.errorsNum, 1)
+        self.assertEqual(r.expectedFailuresNum, 0)
+        self.assertEqual(r.unexpectedSuccessesNum, 0)
+        self.assertFalse(r.wasSuccessful())
+
+    def test_results_aggregate_across_modules(self):
+        # Results combined with "+" must carry the new counters across.
+        class ExpFail(unittest.TestCase):
+            @unittest.expectedFailure
+            def test_x(self):
+                self.assertEqual(1, 0)
+
+        class UnexpSucc(unittest.TestCase):
+            @unittest.expectedFailure
+            def test_x(self):
+                self.assertEqual(1, 1)
+
+        combined = _run(ExpFail) + _run(UnexpSucc)
+        self.assertEqual(combined.expectedFailuresNum, 1)
+        self.assertEqual(combined.unexpectedSuccessesNum, 1)
+        self.assertFalse(combined.wasSuccessful())
+
+
+if __name__ == "__main__":
+    unittest.main()

python-stdlib/unittest/tests/unexpected_success.py

@@ -0,0 +1,14 @@
+# @expectedFailure test that unexpectedly passes -> the run must exit non-zero.
+# Not named "test_*.py" so discovery skips it; tools/ci.sh runs it explicitly.
+
+import unittest
+
+
+class TestUnexpectedSuccess(unittest.TestCase):
+    @unittest.expectedFailure
+    def test_unexpectedly_passes(self):
+        self.assertEqual(1, 1)
+
+
+if __name__ == "__main__":
+    unittest.main()

python-stdlib/unittest/unittest/__init__.py

@@ -196,14 +196,23 @@ def skipUnless(cond, msg):
     return skipIf(not cond, msg)
 
 
+# Sentinels for the two @expectedFailure outcomes: a test that raised
+# (expected failure) vs one that passed (unexpected success).
+class _ExpFail(Exception):
+    pass
+
+
+class _UnexpSucc(Exception):
+    pass
+
+
 def expectedFailure(test):
     def test_exp_fail(*args, **kwargs):
         try:
             test(*args, **kwargs)
-        except:
-            pass
-        else:
-            raise AssertionError("unexpected success")
+        except Exception:
+            raise _ExpFail
+        raise _UnexpSucc
 
     return test_exp_fail
 
@@ -238,13 +247,18 @@ def run(self, suite: TestSuite):
         res.printErrors()
         print("----------------------------------------------------------------------")
         print("Ran %d tests\n" % res.testsRun)
-        if res.failuresNum > 0 or res.errorsNum > 0:
-            print("FAILED (failures=%d, errors=%d)" % (res.failuresNum, res.errorsNum))
-        else:
-            msg = "OK"
-            if res.skippedNum > 0:
-                msg += " (skipped=%d)" % res.skippedNum
-            print(msg)
+        info = []
+        for num, label in (
+            (res.failuresNum, "failures"),
+            (res.errorsNum, "errors"),
+            (res.skippedNum, "skipped"),
+            (res.expectedFailuresNum, "expected failures"),
+            (res.unexpectedSuccessesNum, "unexpected successes"),
+        ):
+            if num:
+                info.append("%s=%d" % (label, num))
+        status = "OK" if res.wasSuccessful() else "FAILED"
+        print("%s (%s)" % (status, ", ".join(info)) if info else status)
 
         return res
 
@@ -257,14 +271,17 @@ def __init__(self):
         self.errorsNum = 0
         self.failuresNum = 0
         self.skippedNum = 0
+        self.expectedFailuresNum = 0
+        self.unexpectedSuccessesNum = 0
         self.testsRun = 0
         self.errors = []
         self.failures = []
         self.skipped = []
         self._newFailures = 0
 
     def wasSuccessful(self):
-        return self.errorsNum == 0 and self.failuresNum == 0
+        # An unexpected success isn't a failure/error but still fails the run.
+        return self.errorsNum == 0 and self.failuresNum == 0 and self.unexpectedSuccessesNum == 0
 
     def printErrors(self):
         if self.errors or self.failures:
@@ -293,6 +310,8 @@ def __add__(self, other):
         self.errorsNum += other.errorsNum
         self.failuresNum += other.failuresNum
         self.skippedNum += other.skippedNum
+        self.expectedFailuresNum += other.expectedFailuresNum
+        self.unexpectedSuccessesNum += other.unexpectedSuccessesNum
         self.testsRun += other.testsRun
         self.errors.extend(other.errors)
         self.failures.extend(other.failures)
@@ -309,6 +328,16 @@ def _handle_test_exception(
         test_result.skipped.append((current_test, reason))
         print(" skipped:", reason)
         return
+    if isinstance(exc, _ExpFail):
+        test_result.expectedFailuresNum += 1
+        if verbose:
+            print(" expected failure")
+        return
+    if isinstance(exc, _UnexpSucc):
+        test_result.unexpectedSuccessesNum += 1
+        if verbose:
+            print(" unexpected success")
+        return
     buf = io.StringIO()
     sys.print_exception(exc, buf)
     ex_str = buf.getvalue()

tools/ci.sh

@@ -111,6 +111,15 @@ function ci_package_tests_run {
         if [ $? -ne 0 ]; then false; return; fi
     done
 
+    # unexpected_success.py isn't named "test_*.py" so discovery skips it; run it
+    # explicitly and assert an unexpected success exits non-zero (like CPython).
+    echo "Running test python-stdlib/unittest/tests/unexpected_success.py (expecting failure)"
+    if (cd python-stdlib/unittest/tests && "${MICROPYTHON}" -m unittest unexpected_success.py); then
+        echo "Error: an unexpected success did not produce a non-zero exit code"
+        false
+        return
+    fi
+
     (cd micropython/usb/usb-device && "${MICROPYTHON}" -m tests.test_core_buffer)
     if [ $? -ne 0 ]; then false; return; fi