Merge branch 'qemu-issue-#21' into 'master'

qemu: added check for timeout and interruption of swap-in processes

Closes #21

See merge request arcfire/rumba!71

1 files changed, 20 insertions, 2 deletions

diff --git a/rumba/testbeds/qemu.py b/rumba/testbeds/qemu.py
index 80a3d4c..4d6804d 100644
--- a/rumba/testbeds/qemu.py
+++ b/rumba/testbeds/qemu.py
@@ -88,6 +88,8 @@ class Testbed(mod.Testbed):
                 errors += 1
                 if not ignore_errors:
                     break
+            except KeyboardInterrupt as e:
+                error_queue.put('Interrupted')
         if errors == 0:
             results_queue.put("Command chain ran correctly")
         else:
@@ -185,6 +187,7 @@ class Testbed(mod.Testbed):
                                  'brctl addif %(br)s %(tap)s'
                                  % {'tap': tap_id, 'br': shim.name}
                                  ).split('\n')
+                command_list += ['sleep 15']
 
                 if shim.link_speed > 0:
                     speed = '%dmbit' % shim.link_speed
@@ -229,14 +232,21 @@ class Testbed(mod.Testbed):
 
         # Wait for all processes to be over.
         total_processes = len(shim_processes)
-        max_waiting_time = 2 * total_processes
+        max_waiting_time = 4 * total_processes
         over_processes = 0
 
-        while max_waiting_time > 0 and over_processes < total_processes:
+        while over_processes < total_processes and max_waiting_time > 0:
+            # Check for deadlock
+
             # Check for errors
             if not e_queue.empty():
                 error_str = str(e_queue.get())
                 logger.error('Testbed instantiation failed: %s', error_str)
+
+                # Wait for the running processes to quit before swapping out
+                for process in shim_processes:
+                    process.join()
+
                 raise Exception('Failure: %s' % error_str)
             try:
                 # Check for results
@@ -248,6 +258,14 @@ class Testbed(mod.Testbed):
             except:
                 max_waiting_time -= 1
 
+        if max_waiting_time == 0:
+            logger.error("Swap in is in deadlock, aborting.")
+            for process in shim_processes:
+                process.terminate()
+                time.sleep(0.1)
+                process.join()
+            raise Exception('Swap in is in deadlock')
+
         for process in shim_processes:
             process.join()