Add ability to check failed tests w/ fragile list

Added ability to check failed tests w/ fragile list to be sure that
the current fail equal to the issue mentioned in the fragile list.
Fragile list should consist of the results files checksums with
mentioned issues in the format:

  fragile = <basename of the test> ; gh-<issue> md5sum:<checksum>

Closes #189.

Author: avtikhon

lib/test.py

@@ -8,6 +8,7 @@
 import sys
 import traceback
 from functools import partial
+from hashlib import md5
 
 try:
     from cStringIO import StringIO
@@ -250,9 +251,12 @@ def run(self, server):
             color_stdout("[ updated ]\n", schema='test_new')
         else:
             has_result = os.path.exists(self.tmp_result)
+            result_checksum = ''
             if has_result:
                 shutil.copy(self.tmp_result, self.reject)
-            short_status = 'fail'
+                with open(self.tmp_result, mode='rb') as result_file:
+                    result_checksum = md5(result_file.read()).hexdigest()
+            short_status = 'fail:' + result_checksum
             color_stdout("[ fail ]\n", schema='test_fail')
 
             where = ""

lib/test_suite.py

@@ -183,6 +183,14 @@ def fragile_tests(self):
                 res.append(test)
         return res
 
+    def fragile_checksums(self):
+        res = []
+        for fragile in self.ini['fragile']:
+            checksum = re.split(r'md5sum:', fragile)
+            if not checksum[0]:
+                res.append(checksum[1])
+        return res
+
     def gen_server(self):
         try:
             return Server(self.ini, test_suite=self)

lib/worker.py

@@ -2,6 +2,7 @@
 import copy
 import functools
 import os
+import re
 import signal
 import traceback
 import yaml
@@ -340,12 +341,15 @@ def run_loop(self, task_queue, result_queue, is_fragile):
                             ' result file checksum "%s" rerunning ...\n'
                             % (task_id[0], task_id[1], short_status), schema='error')
                     short_status = self.run_task(task_id)
+                    fail_checksum = re.split(r'fail:', short_status)
+                    if fragile_checksums and not fail_checksum[1] in fragile_checksums:
+                        break
                     retries_left = retries_left - 1
             else:
                 short_status = self.run_task(task_id)
 
             result_queue.put(self.wrap_result(task_id, short_status))
-            if not lib.Options().args.is_force and short_status == 'fail':
+            if not lib.Options().args.is_force and re.match(r'^fail:.*', short_status):
                 color_stdout(
                     'Worker "%s" got failed test; stopping the server...\n'
                     % self.name, schema='test_var')

listeners.py

@@ -36,13 +36,16 @@ def process_result(self, obj):
         if not isinstance(obj, WorkerTaskResult):
             return
 
-        if obj.short_status not in self.stats:
-            self.stats[obj.short_status] = 0
-        self.stats[obj.short_status] += 1
+        # cut off additional result information from result
+        short_status = obj.short_status.split(':')[0]
+        if short_status not in self.stats:
+            self.stats[short_status] = 0
+        self.stats[short_status] += 1
 
-        if obj.short_status == 'fail':
+        if short_status == 'fail':
             self.failed_tasks.append((obj.task_id,
                                       obj.worker_name,
+                                      obj.short_status.split(':')[1],
                                       obj.show_reproduce_content))
 
     def print_statistics(self):
@@ -56,9 +59,10 @@ def print_statistics(self):
             return False
 
         color_stdout('Failed tasks:\n', schema='test_var')
-        for task_id, worker_name, show_reproduce_content in self.failed_tasks:
+        for task_id, worker_name, md5sum, show_reproduce_content in self.failed_tasks:
             logfile = self.get_logfile(worker_name)
             color_stdout('- %s' % yaml.safe_dump(task_id), schema='test_var')
+            color_stdout('# results file checksum: %s\n' % md5sum)
             color_stdout('# logfile:        %s\n' % logfile)
             reproduce_file_path = get_reproduce_file(worker_name)
             color_stdout('# reproduce file: %s\n' % reproduce_file_path)
@@ -160,7 +164,7 @@ def process_result(self, obj):
         if not isinstance(obj, WorkerTaskResult):
             return
 
-        if obj.short_status == 'fail':
+        if re.match(r'^fail:.*', obj.short_status):
             color_stdout('[Main process] Got failed test; '
                          'gently terminate all workers...\n',
                          schema='test_var')

test-run.py

@@ -51,6 +51,7 @@
 
 import multiprocessing
 import os
+import re
 import sys
 import time
 
@@ -159,7 +160,7 @@ def main_loop_consistent(failed_test_ids):
         worker = task_group['gen_worker'](worker_id)
         for task_id in task_ids:
             short_status = worker.run_task(task_id)
-            if short_status == 'fail':
+            if re.match(r'^fail:.*', short_status):
                 reproduce_file_path = \
                     lib.worker.get_reproduce_file(worker.name)
                 color_stdout('Reproduce file %s\n' %