Add ability to check failed tests w/ fragile list

Added ability to check failed tests w/ fragile list to be sure that
the current fail equal to the issue mentioned in the fragile list.

Part of issue tarantool/tarantool#5050

Author: avtikhon

dispatcher.py

@@ -356,6 +356,7 @@ def __init__(self, key, task_group, randomize):
         self.task_ids = task_group['task_ids']
         self.is_parallel = task_group['is_parallel']
         self.fragile_retries = int(task_group['fragile_retries'])
+        self.fragile_checksums = task_group['fragile_checksums']
         if self.is_parallel:
             self.randomize = randomize
             if self.randomize:
@@ -385,7 +386,8 @@ def _run_worker(self, worker_id, tcp_port_range):
         os.environ['TEST_RUN_TCP_PORT_END'] = str(tcp_port_range[1])
         color_stdout.queue = self.result_queue
         worker = self.gen_worker(worker_id)
-        worker.run_all(self.task_queue, self.result_queue, self.fragile_retries)
+        worker.run_all(self.task_queue, self.result_queue, self.fragile_retries,
+                self.fragile_checksums)
 
     def add_worker(self, worker_id, tcp_port_range):
         # Note: each of our workers should consume only one None, but for the

lib/test.py

@@ -8,6 +8,7 @@
 import sys
 import traceback
 from functools import partial
+from hashlib import md5
 
 try:
     from cStringIO import StringIO
@@ -250,9 +251,12 @@ def run(self, server):
             color_stdout("[ updated ]\n", schema='test_new')
         else:
             has_result = os.path.exists(self.tmp_result)
+            result_checksum = ''
             if has_result:
                 shutil.copy(self.tmp_result, self.reject)
-            short_status = 'fail'
+                with open(self.tmp_result, mode='rb') as result_file:
+                    result_checksum = md5(result_file.read()).hexdigest()
+            short_status = 'fail:' + result_checksum
             color_stdout("[ fail ]\n", schema='test_fail')
 
             where = ""

lib/test_suite.py

@@ -174,6 +174,14 @@ def fragile_tests(self):
                 res.append(test)
         return res
 
+    def fragile_checksums(self):
+        res = []
+        for fragile in self.ini['fragile']:
+            checksum = re.split(r'md5sum:', fragile)
+            if not checksum[0]:
+                res.append(checksum[1])
+        return res
+
     def gen_server(self):
         try:
             return Server(self.ini, test_suite=self)

lib/worker.py

@@ -2,6 +2,7 @@
 import copy
 import functools
 import os
+import re
 import signal
 import traceback
 import yaml
@@ -83,6 +84,7 @@ def get_task_groups():
                 'task_ids': stable_task_ids,
                 'is_parallel': suite.is_parallel(),
                 'fragile_retries': 0,
+                'fragile_checksums': suite.fragile_checksums(),
                 'show_reproduce_content': suite.show_reproduce_content(),
             }
         if fragile_task_ids:
@@ -91,6 +93,7 @@ def get_task_groups():
                 'task_ids': fragile_task_ids,
                 'is_parallel': False,
                 'fragile_retries': suite.fragile_retries(),
+                'fragile_checksums': suite.fragile_checksums(),
                 'show_reproduce_content': suite.show_reproduce_content(),
             }
     return res
@@ -315,7 +318,7 @@ def run_task(self, task_id):
             raise
         return short_status
 
-    def run_loop(self, task_queue, result_queue, fragile_retries):
+    def run_loop(self, task_queue, result_queue, fragile_retries, fragile_checksums):
         """ called from 'run_all' """
         while True:
             task_id = self.task_get(task_queue)
@@ -332,15 +335,17 @@ def run_loop(self, task_queue, result_queue, fragile_retries):
             # 'fragile_retries' value comes from the suite configuration
             retries_left = fragile_retries
             while short_status != 'pass' and retries_left >= 0:
-                if short_status == 'fail':
+                if re.match(r'^fail:.*', short_status):
                     color_stdout(
                         'Test "%s", conf: "%s"\n'
-                        '\tfrom "fragile" list failed - rerunning ...\n'
-                        % (task_id[0], task_id[1]) , schema='error')
+                        '\tfrom "fragile" list failed, result file checksum "%s" rerunning ...\n'
+                        % (task_id[0], task_id[1], short_status), schema='error')
                 short_status = self.run_task(task_id)
+                fail_checksum = re.split(r'fail:', short_status)
+                if fragile_checksums and not fail_checksum[1] in fragile_checksums:
+                    break
                 retries_left = retries_left - 1
 
-
             result_queue.put(self.wrap_result(task_id, short_status))
             if not lib.Options().args.is_force and short_status == 'fail':
                 color_stdout(
@@ -354,14 +359,14 @@ def run_loop(self, task_queue, result_queue, fragile_retries):
                 raise VoluntaryStopException()
             self.task_done(task_queue)
 
-    def run_all(self, task_queue, result_queue, fragile_retries):
+    def run_all(self, task_queue, result_queue, fragile_retries, fragile_checksums):
         if not self.initialized:
             self.flush_all_tasks(task_queue, result_queue)
             result_queue.put(self.done_marker())
             return
 
         try:
-            self.run_loop(task_queue, result_queue, fragile_retries)
+            self.run_loop(task_queue, result_queue, fragile_retries, fragile_checksums)
         except (KeyboardInterrupt, Exception) as e:
             if not isinstance(e, KeyboardInterrupt) and \
                not isinstance(e, VoluntaryStopException):