Collect results artifacts of the failed workers

ArtifactsWatcher listener collects list of all workers with failed
tests. After overall testing finishes it copies workers artifacts
files from its running 'vardir' sub-directories to the common path
'<vardir>/artifacts' to be able to collect these artifacts later.

Closes #90

dispatcher.py

@@ -95,6 +95,7 @@ def __init__(self, task_groups, max_workers_cnt, randomize):
         self.report_timeout = 1.0
 
         self.statistics = None
+        self.artifacts = None
         self.fail_watcher = None
         self.listeners = None
         self.init_listeners()
@@ -132,8 +133,10 @@ def init_listeners(self):
         log_output_watcher = listeners.LogOutputWatcher()
         self.statistics = listeners.StatisticsWatcher(
             log_output_watcher.get_logfile)
+        self.artifacts = listeners.ArtifactsWatcher(
+            log_output_watcher.get_logfile)
         output_watcher = listeners.OutputWatcher()
-        self.listeners = [self.statistics, log_output_watcher, output_watcher]
+        self.listeners = [self.statistics, log_output_watcher, output_watcher, self.artifacts]
         if watch_fail:
             self.fail_watcher = listeners.FailWatcher(
                 self.terminate_all_workers)

listeners.py

@@ -2,6 +2,7 @@
 import re
 import sys
 import yaml
+import shutil
 
 import lib
 from lib.colorer import color_stdout
@@ -11,6 +12,7 @@
 from lib.worker import WorkerTaskResult
 from lib.worker import get_reproduce_file
 from lib.utils import prefix_each_line
+from lib.utils import safe_makedirs
 
 
 class BaseWatcher(object):
@@ -70,6 +72,52 @@ def print_statistics(self):
         return True
 
 
+class ArtifactsWatcher(BaseWatcher):
+    """ArtifactsWatcher listener collects list of all workers with failed
+    tests. After overall testing finishes it copies workers artifacts
+    files from its running 'vardir' sub-directories to the common path
+    '<vardir>/artifacts' to be able to collect these artifacts later.
+    """
+    def __init__(self, get_logfile):
+        self.failed_workers = []
+        self.get_logfile = get_logfile
+
+    def process_result(self, obj):
+        if not isinstance(obj, WorkerTaskResult):
+            return
+
+        if obj.short_status == 'fail' and \
+                obj.worker_name not in self.failed_workers:
+            self.failed_workers.append(obj.worker_name)
+
+    def save_artifacts(self):
+        if not self.failed_workers:
+            return
+
+        vardir = lib.Options().args.vardir
+        artifacts_dir = os.path.join(vardir, 'artifacts')
+        artifacts_log_dir = os.path.join(artifacts_dir, 'log')
+        artifacts_reproduce_dir = os.path.join(artifacts_dir, 'reproduce')
+        safe_makedirs(artifacts_dir)
+        safe_makedirs(artifacts_log_dir)
+        safe_makedirs(artifacts_reproduce_dir)
+
+        for worker_name in self.failed_workers:
+            logfile = self.get_logfile(worker_name)
+            reproduce_file_path = get_reproduce_file(worker_name)
+            shutil.copy(logfile,
+                        os.path.join(artifacts_log_dir,
+                                     os.path.basename(logfile)))
+            shutil.copy(reproduce_file_path,
+                        os.path.join(artifacts_reproduce_dir,
+                                     os.path.basename(reproduce_file_path)))
+            shutil.copytree(os.path.join(vardir, worker_name),
+                            os.path.join(artifacts_dir, worker_name),
+                            ignore=shutil.ignore_patterns(
+                                '*.socket-iproto', '*.socket-admin',
+                                '*.sock', '*.control'))
+
+
 class LogOutputWatcher(BaseWatcher):
     def __init__(self):
         self.fds = dict()

test-run.py

@@ -110,6 +110,7 @@ def main_loop_parallel():
         has_undone = dispatcher.report_undone(
             verbose=bool(is_force or not has_failed))
         if has_failed:
+            dispatcher.artifacts.save_artifacts()
             return EXIT_FAILED_TEST
         if has_undone:
             return EXIT_NOTDONE_TEST