utils & storyboard: add syslog retrieval functionality

Implements #39.
Also updated examples.

8 files changed, 167 insertions, 36 deletions

diff --git a/examples/converged-operator-network.py b/examples/converged-operator-network.py
index 6242e31..3b80746 100755
--- a/examples/converged-operator-network.py
+++ b/examples/converged-operator-network.py
@@ -5,7 +5,7 @@
 
 from rumba.model import *
 from rumba.storyboard import *
-from rumba.utils import ExperimentManager
+from rumba.utils import *
 
 # import testbed plugins
 import rumba.testbeds.emulab as emulab
@@ -186,7 +186,7 @@ exp = rl.Experiment(tb, nodes = [f1n1, f1n2, f1n3, f1n4,
 
 print(exp)
 
-with ExperimentManager(exp, do_swap_out=ExperimentManager.PROMPT):
+with ExperimentManager(exp, swap_out_strategy=PAUSE_SWAPOUT):
     exp.swap_in()
     exp.install_prototype()
     exp.bootstrap_prototype()
diff --git a/examples/scalingtime.py b/examples/scalingtime.py
index 4a35d67..f0c9285 100755
--- a/examples/scalingtime.py
+++ b/examples/scalingtime.py
@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 from rumba.model import *
-from rumba.utils import ExperimentManager
+from rumba.utils import *
 
 # import testbed plugins
 import rumba.testbeds.emulab as emulab
@@ -127,7 +127,7 @@ tb = jfed.Testbed(exp_name = "metro2",
 
 exp = our.Experiment(tb, nodes = nodes)
 
-with ExperimentManager(exp, do_swap_out = ExperimentManager.PROMPT):
+with ExperimentManager(exp, swap_out_strategy=PAUSE_SWAPOUT):
     exp.swap_in()
     exp.install_prototype()
     exp.bootstrap_prototype()
diff --git a/rumba/model.py b/rumba/model.py
index 5647b1e..2749ae7 100644
--- a/rumba/model.py
+++ b/rumba/model.py
@@ -384,20 +384,22 @@ class Node(object):
             destination
         )
 
-    def fetch_files(self, paths, destination):
+    def fetch_files(self, paths, destination, sudo=False):
         ssh_support.copy_files_from_testbed(
             self.ssh_config,
             self.ssh_config,
             paths,
-            destination
+            destination,
+            sudo=sudo
         )
 
-    def fetch_file(self, path, destination):
+    def fetch_file(self, path, destination, sudo=False):
         ssh_support.copy_file_from_testbed(
             self.ssh_config,
             self.ssh_config,
             path,
-            destination
+            destination,
+            sudo=sudo
         )
 
 
@@ -562,7 +564,7 @@ class Experiment(object):
             # If it is None, use /tmp/rumba/{project}
             # Wipe it and make it again
             exp_name = self.testbed.exp_name.replace('/', '_')  # Just in case
-            log_dir = '/tmp/rumba/' + exp_name + '/'
+            log_dir = os.path.join(tmp_dir, exp_name)
             shutil.rmtree(log_dir, ignore_errors=True)
             os.mkdir(log_dir)
         self.log_dir = log_dir
diff --git a/rumba/prototypes/irati.py b/rumba/prototypes/irati.py
index 41de9bb..335371e 100644
--- a/rumba/prototypes/irati.py
+++ b/rumba/prototypes/irati.py
@@ -62,7 +62,11 @@ class Experiment(mod.Experiment):
     def __init__(self, testbed, nodes=None,
                  git_repo='https://github.com/IRATI/stack',
                  git_branch='arcfire', installpath=None, varpath=None):
-        mod.Experiment.__init__(self, testbed, nodes, git_repo, git_branch)
+        mod.Experiment.__init__(self,
+                                testbed,
+                                nodes,
+                                git_repo,
+                                git_branch)
         if installpath is None:
             installpath = '/usr'
         if varpath is None:
diff --git a/rumba/ssh_support.py b/rumba/ssh_support.py
index 69c049c..98caca2 100644
--- a/rumba/ssh_support.py
+++ b/rumba/ssh_support.py
@@ -37,12 +37,14 @@ try:
 except NameError:
     pass
 
+
 logger = log.get_logger(__name__)
 
 
 class SSHException(Exception):
     pass
 
+
 def get_ssh_client():
     ssh_client = paramiko.SSHClient()
     ssh_client.load_system_host_keys()
@@ -64,6 +66,7 @@ def _print_stream(stream):
             o_array = []
         return '\n'.join(o_array)
 
+
 def ssh_connect(hostname, port, username, password, time_out, proxy_server):
     logger.debug('Trying to open a connection towards node %s.' % hostname)
     retry = 0
@@ -97,7 +100,7 @@ def ssh_connect(hostname, port, username, password, time_out, proxy_server):
             logger.error('If you are sure this is not a man in the ' +
                          'middle attack, edit that file to remove the ' +
                          'entry and then hit return to try again.')
-            input()
+            input('Key mismatch detected. Press ENTER when ready.')
         except (paramiko.ssh_exception.SSHException, EOFError):
             retry += 1
             logger.error('Failed to connect to host, retrying: ' +
@@ -105,6 +108,7 @@ def ssh_connect(hostname, port, username, password, time_out, proxy_server):
     if retry == max_retries:
         raise SSHException('Failed to connect to host')
 
+
 def execute_proxy_commands(testbed, ssh_config, commands, time_out=3):
     """
     Remote execution of a list of shell command on hostname, using the
@@ -202,6 +206,7 @@ def execute_commands(testbed, ssh_config, commands, time_out=3):
                                '\n\t'.join(list_print) + '\n')
     return o
 
+
 def execute_command(testbed, ssh_config, command, time_out=3):
     """
     Remote execution of a list of shell command on hostname. By
@@ -256,6 +261,7 @@ def write_text_to_file(testbed, ssh_config, text, file_name):
     except SSHException as e:
         raise SSHException('Failed to write text to remote file')
 
+
 def copy_files_to_testbed(testbed, ssh_config, paths, destination):
     """
     Copies local files to a remote node.
@@ -292,6 +298,7 @@ def copy_files_to_testbed(testbed, ssh_config, paths, destination):
     except Exception as e:
         raise SSHException('Failed to copy files to testbed')
 
+
 def copy_file_to_testbed(testbed, ssh_config, path, destination):
     """
     Copies a local file to a remote node.
@@ -304,7 +311,8 @@ def copy_file_to_testbed(testbed, ssh_config, path, destination):
     copy_files_to_testbed(testbed, ssh_config, [path], destination)
 
 
-def copy_files_from_testbed(testbed, ssh_config, paths, destination):
+def copy_files_from_testbed(testbed, ssh_config, paths,
+                            destination, sudo=False):
     """
     Copies local files to a remote node.
 
@@ -312,10 +320,15 @@ def copy_files_from_testbed(testbed, ssh_config, paths, destination):
     @param ssh_config: ssh config of the node
     @param paths: source paths (remote) as an iterable
     @param destination: destination folder name (local)
+    @param sudo: if path to copy requires root access, should be set to true
     """
     if destination is not '' and not destination.endswith('/'):
         destination = destination + '/'
 
+    if sudo:
+        execute_command(testbed, ssh_config,
+                        'sudo chmod a+rw %s' % (" ".join(paths)))
+
     if ssh_config.client is None:
         client, proxy_client = ssh_connect(ssh_config.hostname, ssh_config.port,
                                            testbed.username, testbed.password,
@@ -341,7 +354,8 @@ def copy_files_from_testbed(testbed, ssh_config, paths, destination):
         raise SSHException('Failed to copy files from testbed', e)
 
 
-def copy_file_from_testbed(testbed, ssh_config, path, destination):
+def copy_file_from_testbed(testbed, ssh_config, path,
+                           destination, sudo=False):
     """
     Copies a local file to a remote node.
 
@@ -349,8 +363,9 @@ def copy_file_from_testbed(testbed, ssh_config, path, destination):
     @param ssh_config: ssh config of the node
     @param path: source path (remote)
     @param destination: destination folder name (local)
+    @param sudo: if path to copy requires root access, should be set to true
     """
-    copy_files_from_testbed(testbed, ssh_config, [path], destination)
+    copy_files_from_testbed(testbed, ssh_config, [path], destination, sudo)
 
 
 def setup_vlans(testbed, node, vlans):
diff --git a/rumba/storyboard.py b/rumba/storyboard.py
index 26e91fd..ca0cfb5 100644
--- a/rumba/storyboard.py
+++ b/rumba/storyboard.py
@@ -229,7 +229,7 @@ class Server:
 
     def run(self):
         for node in self.nodes:
-            logfile = "%s_server.log" % self.ap
+            logfile = "/tmp/%s_server.log" % self.ap
             script = r'nohup "$@" > %s 2>&1 & echo "$!"' % (logfile,)
             run_cmd = self.ap + (
                 (" " + self.options) if self.options is not None else ""
@@ -434,18 +434,21 @@ class StoryBoard:
             for server in self.servers:
                 server.stop()
 
-    def fetch_logs(self, local_dir='.'):
+    def fetch_logs(self, local_dir=None):
+        if local_dir is None:
+            local_dir = self.experiment.log_dir
         if not os.path.isdir(local_dir):
-            raise Exception('"%s" is not a directory. Cannot fetch logs.'
+            raise Exception('Destination "%s" is not a directory. '
+                            'Cannot fetch logs.'
                             % local_dir)
         for node in self.server_nodes:
-            logs_list = node.execute_command('ls *_server.log')
+            logs_list = node.execute_command('ls /tmp/*_server.log')
             logs_list = [x for x in logs_list.split('\n') if x != '']
-            logger.info('Log list is:\n%s', logs_list)
+            logger.debug('Log list is:\n%s', logs_list)
             node.fetch_files(logs_list, local_dir)
         for node in self.client_nodes:
             logs_list = node.execute_command('ls /tmp/*.rumba.log '
                                              '|| echo ""')
             logs_list = [x for x in logs_list.split('\n') if x != '']
-            logger.info('Log list is:\n%s', logs_list)
+            logger.debug('Log list is:\n%s', logs_list)
             node.fetch_files(logs_list, local_dir)
diff --git a/rumba/utils.py b/rumba/utils.py
index 2a8c6b7..78a80aa 100644
--- a/rumba/utils.py
+++ b/rumba/utils.py
@@ -23,8 +23,8 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., http://www.fsf.org/about/contact/.
 #
-
-import time
+import enum
+import os
 
 import rumba.log as log
 import rumba.model as model
@@ -38,31 +38,133 @@ except NameError:
 logger = log.get_logger(__name__)
 
 
-class ExperimentManager(object):
+class SwapOutStrategy(enum.Enum):
+
+    NO = 0
+    AUTO = 1
+    PAUSE = 2
+    PROMPT = 3
+
+
+class SyslogsStrategy(enum.Enum):
+
+    NO = 0
+    DEFAULT = 1
+    DMESG = 2
+    CUSTOM = 3
+
+
+# Utility names for importing in the scripts
+NO_SWAPOUT = SwapOutStrategy.NO
+AUTO_SWAPOUT = SwapOutStrategy.AUTO
+PAUSE_SWAPOUT = SwapOutStrategy.PAUSE
+PROMPT_SWAPOUT = SwapOutStrategy.PROMPT
+
+NO_SYSLOGS = SyslogsStrategy.NO
+DEFAULT_SYSLOGS = SyslogsStrategy.DEFAULT
+DMESG_SYSLOGS = SyslogsStrategy.DMESG
+CUSTOM_SYSLOGS = SyslogsStrategy.CUSTOM
 
-    PROMPT = 1
-    AUTO = 2
-    NO = 3
 
-    def __init__(self, experiment, do_swap_out=AUTO):
+class ExperimentManager(object):
+
+    def __init__(self,
+                 experiment,
+                 swap_out_strategy=AUTO_SWAPOUT,
+                 syslogs_strategy=NO_SYSLOGS,
+                 syslogs=None):
         assert isinstance(experiment, model.Experiment), \
             'An experiment instance is required.'
         self.experiment = experiment
-        self.do_swap_out = do_swap_out
+        self.swap_out_strategy = swap_out_strategy
+        self.syslogs_strategy = syslogs_strategy
+        self.syslogs = [syslogs] if isinstance(syslogs, str) else syslogs
+        self.use_sudo = self.experiment.testbed.username != 'root'
 
     def __enter__(self):
         pass
 
+    def fetch_dmesg_syslog(self, node, node_dir):
+        node.execute_command('dmesg > /tmp/dmesg')
+        node.fetch_file('/tmp/dmesg', node_dir)
+
+    def fetch_syslog(self, node, node_dir):
+        node.fetch_files(self.syslogs,
+                         node_dir, self.use_sudo)
+
+    def fetch_syslogs(self):
+        local_dir = self.experiment.log_dir
+
+        # Define and set up fetching function
+        if self.syslogs_strategy == DMESG_SYSLOGS:
+            fetching_function = self.fetch_dmesg_syslog
+        elif self.syslogs_strategy == DEFAULT_SYSLOGS:
+            self.syslogs = self.experiment.testbed.system_logs
+            fetching_function = self.fetch_syslog
+        elif self.syslogs_strategy == CUSTOM_SYSLOGS:
+            assert self.syslogs is not None, \
+                'Custom syslog strategy requires specifying a path'
+            fetching_function = self.fetch_syslog
+        else:
+            raise ValueError('Unknown syslogs strategy %s'
+                             % self.syslogs_strategy)
+
+        for node in self.experiment.nodes:
+            node_dir = os.path.join(local_dir, node.name)
+            if not os.path.isdir(node_dir):
+                os.mkdir(node_dir)
+            try:
+                fetching_function(node, node_dir)
+            except Exception as e:
+                logger.warning(
+                    'Could not fetch syslogs of node %s. %s%s',
+                    node.name,
+                    type(e).__name__,
+                    (": " + str(e)) if str(e) != "" else ""
+                )
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         try:
-            if self.do_swap_out == self.PROMPT:
-                logger.info('Press ENTER to start swap out.')
-                input('')
-            if self.do_swap_out == self.PROMPT \
-                    or self.do_swap_out == self.AUTO:
+            # Pause to let the user play
+            if self.swap_out_strategy == PAUSE_SWAPOUT:
+                input('Press ENTER to start swap out.')
+                do_swap_out = True
+            elif self.swap_out_strategy == PROMPT_SWAPOUT:
+                do_swap_out = None
+                while do_swap_out is None:
+                    ans = input('Swap out experiment? (y/n): ')
+                    if ans == 'y':
+                        do_swap_out = True
+                    elif ans == 'n':
+                        do_swap_out = False
+                    else:
+                        print("Only 'y' or 'n' please.")
+            elif self.swap_out_strategy == AUTO_SWAPOUT:
+                do_swap_out = True
+            elif self.swap_out_strategy == NO_SWAPOUT:
+                do_swap_out = False
+            else:
+                logger.warning('Unknown swap-out strategy %s. Swapping out.',
+                               self.swap_out_strategy)
+                do_swap_out = True
+
+            # Fetch syslogs (if requested)
+            if self.syslogs_strategy != NO_SYSLOGS:
+                try:
+                    self.fetch_syslogs()
+                except Exception as e:
+                    logger.warning(
+                        'There has been a problem fetching syslogs. %s%s',
+                        type(e).__name__,
+                        ": " + str(e) if str(e) != "" else ""
+                    )
+
+            # Swap out
+            if do_swap_out:
                 self.experiment.swap_out()
             if exc_val is not None:
-                logger.error('Something went wrong. Got %s: %s',
+                logger.error('Something went wrong during swap out. '
+                             'Got %s: %s',
                              type(exc_val).__name__, str(exc_val))
                 logger.debug('Exception details:', exc_info=exc_val)
         finally:
@@ -70,4 +172,4 @@ class ExperimentManager(object):
             # Make sure to print all logs before execution terminates,
             # Specifically the last two error logs above.
         return True
-        # Suppress the exception we logged: no traceback, unless requested.
+        # Suppress the exception we logged: no traceback, unless logged.
diff --git a/setup.py b/setup.py
index c70cf6a..99ff69c 100755
--- a/setup.py
+++ b/setup.py
@@ -13,7 +13,12 @@ setuptools.setup(
     license='LGPL',
     description='Rumba measurement framework for RINA',
     packages=['rumba', 'rumba.testbeds', 'rumba.prototypes'],
-    install_requires=['paramiko', 'repoze.lru; python_version<"3.2"', 'contextlib2; python_version<"3.0"'],
+    install_requires=[
+        'paramiko',
+        'repoze.lru; python_version<"3.2"',
+        'contextlib2; python_version<"3.0"',
+        'enum34; python_version<"3.0"'
+    ],
     extras_require={'NumpyAcceleration': ['numpy']},
     scripts=['tools/rumba-access']
 )