From 19ea30bdd551fedab493f26d5d6c33a6b39bb49b Mon Sep 17 00:00:00 2001
From: Damien George <damien.p.george@gmail.com>
Date: Wed, 18 Mar 2020 21:33:17 +1100
Subject: [PATCH] tests/run-multitests.py: Print test summary and do exit(1) on
 failure.

---
 tests/run-multitests.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/tests/run-multitests.py b/tests/run-multitests.py
index f27af3b0e..edf2238a5 100755
--- a/tests/run-multitests.py
+++ b/tests/run-multitests.py
@@ -303,6 +303,10 @@ def run_test_on_instances(test_file, num_instances, instances):
 
 
 def run_tests(test_files, instances_truth, instances_test):
+    skipped_tests = []
+    passed_tests = []
+    failed_tests = []
+
     for test_file, num_instances in test_files:
         instances_str = "|".join(str(instances_test[i]) for i in range(num_instances))
         print("{} on {}: ".format(test_file, instances_str), end="")
@@ -335,10 +339,13 @@ def run_tests(test_files, instances_truth, instances_test):
         # Print result of test
         if skip:
             print("skip")
+            skipped_tests.append(test_file)
         elif output_test == output_truth:
             print("pass")
+            passed_tests.append(test_file)
         else:
             print("FAIL")
+            failed_tests.append(test_file)
             if not cmd_args.show_output:
                 print("### TEST ###")
                 print(output_test, end="")
@@ -348,6 +355,16 @@ def run_tests(test_files, instances_truth, instances_test):
         if cmd_args.show_output:
             print()
 
+    print("{} tests performed".format(len(skipped_tests) + len(passed_tests) + len(failed_tests)))
+    print("{} tests passed".format(len(passed_tests)))
+
+    if skipped_tests:
+        print("{} tests skipped: {}".format(len(skipped_tests), " ".join(skipped_tests)))
+    if failed_tests:
+        print("{} tests failed: {}".format(len(failed_tests), " ".join(failed_tests)))
+
+    return not failed_tests
+
 
 def main():
     global cmd_args
@@ -384,13 +401,16 @@ def main():
         instances_test.append(PyInstanceSubProcess([MICROPYTHON]))
 
     try:
-        run_tests(test_files, instances_truth, instances_test)
+        all_pass = run_tests(test_files, instances_truth, instances_test)
     finally:
         for i in instances_truth:
             i.close()
         for i in instances_test:
             i.close()
 
+    if not all_pass:
+        sys.exit(1)
+
 
 if __name__ == "__main__":
     main()