utils & storyboard: add syslog retrieval functionality

Implements #39.
Also updated examples.

1 files changed, 117 insertions, 15 deletions

diff --git a/rumba/utils.py b/rumba/utils.py
index 2a8c6b7..78a80aa 100644
--- a/rumba/utils.py
+++ b/rumba/utils.py
@@ -23,8 +23,8 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., http://www.fsf.org/about/contact/.
 #
-
-import time
+import enum
+import os
 
 import rumba.log as log
 import rumba.model as model
@@ -38,31 +38,133 @@ except NameError:
 logger = log.get_logger(__name__)
 
 
-class ExperimentManager(object):
+class SwapOutStrategy(enum.Enum):
+
+    NO = 0
+    AUTO = 1
+    PAUSE = 2
+    PROMPT = 3
+
+
+class SyslogsStrategy(enum.Enum):
+
+    NO = 0
+    DEFAULT = 1
+    DMESG = 2
+    CUSTOM = 3
+
+
+# Utility names for importing in the scripts
+NO_SWAPOUT = SwapOutStrategy.NO
+AUTO_SWAPOUT = SwapOutStrategy.AUTO
+PAUSE_SWAPOUT = SwapOutStrategy.PAUSE
+PROMPT_SWAPOUT = SwapOutStrategy.PROMPT
+
+NO_SYSLOGS = SyslogsStrategy.NO
+DEFAULT_SYSLOGS = SyslogsStrategy.DEFAULT
+DMESG_SYSLOGS = SyslogsStrategy.DMESG
+CUSTOM_SYSLOGS = SyslogsStrategy.CUSTOM
 
-    PROMPT = 1
-    AUTO = 2
-    NO = 3
 
-    def __init__(self, experiment, do_swap_out=AUTO):
+class ExperimentManager(object):
+
+    def __init__(self,
+                 experiment,
+                 swap_out_strategy=AUTO_SWAPOUT,
+                 syslogs_strategy=NO_SYSLOGS,
+                 syslogs=None):
         assert isinstance(experiment, model.Experiment), \
             'An experiment instance is required.'
         self.experiment = experiment
-        self.do_swap_out = do_swap_out
+        self.swap_out_strategy = swap_out_strategy
+        self.syslogs_strategy = syslogs_strategy
+        self.syslogs = [syslogs] if isinstance(syslogs, str) else syslogs
+        self.use_sudo = self.experiment.testbed.username != 'root'
 
     def __enter__(self):
         pass
 
+    def fetch_dmesg_syslog(self, node, node_dir):
+        node.execute_command('dmesg > /tmp/dmesg')
+        node.fetch_file('/tmp/dmesg', node_dir)
+
+    def fetch_syslog(self, node, node_dir):
+        node.fetch_files(self.syslogs,
+                         node_dir, self.use_sudo)
+
+    def fetch_syslogs(self):
+        local_dir = self.experiment.log_dir
+
+        # Define and set up fetching function
+        if self.syslogs_strategy == DMESG_SYSLOGS:
+            fetching_function = self.fetch_dmesg_syslog
+        elif self.syslogs_strategy == DEFAULT_SYSLOGS:
+            self.syslogs = self.experiment.testbed.system_logs
+            fetching_function = self.fetch_syslog
+        elif self.syslogs_strategy == CUSTOM_SYSLOGS:
+            assert self.syslogs is not None, \
+                'Custom syslog strategy requires specifying a path'
+            fetching_function = self.fetch_syslog
+        else:
+            raise ValueError('Unknown syslogs strategy %s'
+                             % self.syslogs_strategy)
+
+        for node in self.experiment.nodes:
+            node_dir = os.path.join(local_dir, node.name)
+            if not os.path.isdir(node_dir):
+                os.mkdir(node_dir)
+            try:
+                fetching_function(node, node_dir)
+            except Exception as e:
+                logger.warning(
+                    'Could not fetch syslogs of node %s. %s%s',
+                    node.name,
+                    type(e).__name__,
+                    (": " + str(e)) if str(e) != "" else ""
+                )
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         try:
-            if self.do_swap_out == self.PROMPT:
-                logger.info('Press ENTER to start swap out.')
-                input('')
-            if self.do_swap_out == self.PROMPT \
-                    or self.do_swap_out == self.AUTO:
+            # Pause to let the user play
+            if self.swap_out_strategy == PAUSE_SWAPOUT:
+                input('Press ENTER to start swap out.')
+                do_swap_out = True
+            elif self.swap_out_strategy == PROMPT_SWAPOUT:
+                do_swap_out = None
+                while do_swap_out is None:
+                    ans = input('Swap out experiment? (y/n): ')
+                    if ans == 'y':
+                        do_swap_out = True
+                    elif ans == 'n':
+                        do_swap_out = False
+                    else:
+                        print("Only 'y' or 'n' please.")
+            elif self.swap_out_strategy == AUTO_SWAPOUT:
+                do_swap_out = True
+            elif self.swap_out_strategy == NO_SWAPOUT:
+                do_swap_out = False
+            else:
+                logger.warning('Unknown swap-out strategy %s. Swapping out.',
+                               self.swap_out_strategy)
+                do_swap_out = True
+
+            # Fetch syslogs (if requested)
+            if self.syslogs_strategy != NO_SYSLOGS:
+                try:
+                    self.fetch_syslogs()
+                except Exception as e:
+                    logger.warning(
+                        'There has been a problem fetching syslogs. %s%s',
+                        type(e).__name__,
+                        ": " + str(e) if str(e) != "" else ""
+                    )
+
+            # Swap out
+            if do_swap_out:
                 self.experiment.swap_out()
             if exc_val is not None:
-                logger.error('Something went wrong. Got %s: %s',
+                logger.error('Something went wrong during swap out. '
+                             'Got %s: %s',
                              type(exc_val).__name__, str(exc_val))
                 logger.debug('Exception details:', exc_info=exc_val)
         finally:
@@ -70,4 +172,4 @@ class ExperimentManager(object):
             # Make sure to print all logs before execution terminates,
             # Specifically the last two error logs above.
         return True
-        # Suppress the exception we logged: no traceback, unless requested.
+        # Suppress the exception we logged: no traceback, unless logged.