diff options
| -rwxr-xr-x | examples/converged-operator-network.py | 4 | ||||
| -rwxr-xr-x | examples/scalingtime.py | 4 | ||||
| -rw-r--r-- | rumba/model.py | 12 | ||||
| -rw-r--r-- | rumba/prototypes/irati.py | 6 | ||||
| -rw-r--r-- | rumba/ssh_support.py | 23 | ||||
| -rw-r--r-- | rumba/storyboard.py | 15 | ||||
| -rw-r--r-- | rumba/utils.py | 132 | ||||
| -rwxr-xr-x | setup.py | 7 |
8 files changed, 167 insertions, 36 deletions
diff --git a/examples/converged-operator-network.py b/examples/converged-operator-network.py index 6242e31..3b80746 100755 --- a/examples/converged-operator-network.py +++ b/examples/converged-operator-network.py @@ -5,7 +5,7 @@ from rumba.model import * from rumba.storyboard import * -from rumba.utils import ExperimentManager +from rumba.utils import * # import testbed plugins import rumba.testbeds.emulab as emulab @@ -186,7 +186,7 @@ exp = rl.Experiment(tb, nodes = [f1n1, f1n2, f1n3, f1n4, print(exp) -with ExperimentManager(exp, do_swap_out=ExperimentManager.PROMPT): +with ExperimentManager(exp, swap_out_strategy=PAUSE_SWAPOUT): exp.swap_in() exp.install_prototype() exp.bootstrap_prototype() diff --git a/examples/scalingtime.py b/examples/scalingtime.py index 4a35d67..f0c9285 100755 --- a/examples/scalingtime.py +++ b/examples/scalingtime.py @@ -1,7 +1,7 @@ #!/usr/bin/env python from rumba.model import * -from rumba.utils import ExperimentManager +from rumba.utils import * # import testbed plugins import rumba.testbeds.emulab as emulab @@ -127,7 +127,7 @@ tb = jfed.Testbed(exp_name = "metro2", exp = our.Experiment(tb, nodes = nodes) -with ExperimentManager(exp, do_swap_out = ExperimentManager.PROMPT): +with ExperimentManager(exp, swap_out_strategy=PAUSE_SWAPOUT): exp.swap_in() exp.install_prototype() exp.bootstrap_prototype() diff --git a/rumba/model.py b/rumba/model.py index 5647b1e..2749ae7 100644 --- a/rumba/model.py +++ b/rumba/model.py @@ -384,20 +384,22 @@ class Node(object): destination ) - def fetch_files(self, paths, destination): + def fetch_files(self, paths, destination, sudo=False): ssh_support.copy_files_from_testbed( self.ssh_config, self.ssh_config, paths, - destination + destination, + sudo=sudo ) - def fetch_file(self, path, destination): + def fetch_file(self, path, destination, sudo=False): ssh_support.copy_file_from_testbed( self.ssh_config, self.ssh_config, path, - destination + destination, + sudo=sudo ) @@ -562,7 +564,7 @@ class Experiment(object): # If it is None, use /tmp/rumba/{project} # Wipe it and make it again exp_name = self.testbed.exp_name.replace('/', '_') # Just in case - log_dir = '/tmp/rumba/' + exp_name + '/' + log_dir = os.path.join(tmp_dir, exp_name) shutil.rmtree(log_dir, ignore_errors=True) os.mkdir(log_dir) self.log_dir = log_dir diff --git a/rumba/prototypes/irati.py b/rumba/prototypes/irati.py index 41de9bb..335371e 100644 --- a/rumba/prototypes/irati.py +++ b/rumba/prototypes/irati.py @@ -62,7 +62,11 @@ class Experiment(mod.Experiment): def __init__(self, testbed, nodes=None, git_repo='https://github.com/IRATI/stack', git_branch='arcfire', installpath=None, varpath=None): - mod.Experiment.__init__(self, testbed, nodes, git_repo, git_branch) + mod.Experiment.__init__(self, + testbed, + nodes, + git_repo, + git_branch) if installpath is None: installpath = '/usr' if varpath is None: diff --git a/rumba/ssh_support.py b/rumba/ssh_support.py index 69c049c..98caca2 100644 --- a/rumba/ssh_support.py +++ b/rumba/ssh_support.py @@ -37,12 +37,14 @@ try: except NameError: pass + logger = log.get_logger(__name__) class SSHException(Exception): pass + def get_ssh_client(): ssh_client = paramiko.SSHClient() ssh_client.load_system_host_keys() @@ -64,6 +66,7 @@ def _print_stream(stream): o_array = [] return '\n'.join(o_array) + def ssh_connect(hostname, port, username, password, time_out, proxy_server): logger.debug('Trying to open a connection towards node %s.' % hostname) retry = 0 @@ -97,7 +100,7 @@ def ssh_connect(hostname, port, username, password, time_out, proxy_server): logger.error('If you are sure this is not a man in the ' + 'middle attack, edit that file to remove the ' + 'entry and then hit return to try again.') - input() + input('Key mismatch detected. Press ENTER when ready.') except (paramiko.ssh_exception.SSHException, EOFError): retry += 1 logger.error('Failed to connect to host, retrying: ' + @@ -105,6 +108,7 @@ def ssh_connect(hostname, port, username, password, time_out, proxy_server): if retry == max_retries: raise SSHException('Failed to connect to host') + def execute_proxy_commands(testbed, ssh_config, commands, time_out=3): """ Remote execution of a list of shell command on hostname, using the @@ -202,6 +206,7 @@ def execute_commands(testbed, ssh_config, commands, time_out=3): '\n\t'.join(list_print) + '\n') return o + def execute_command(testbed, ssh_config, command, time_out=3): """ Remote execution of a list of shell command on hostname. By @@ -256,6 +261,7 @@ def write_text_to_file(testbed, ssh_config, text, file_name): except SSHException as e: raise SSHException('Failed to write text to remote file') + def copy_files_to_testbed(testbed, ssh_config, paths, destination): """ Copies local files to a remote node. @@ -292,6 +298,7 @@ def copy_files_to_testbed(testbed, ssh_config, paths, destination): except Exception as e: raise SSHException('Failed to copy files to testbed') + def copy_file_to_testbed(testbed, ssh_config, path, destination): """ Copies a local file to a remote node. @@ -304,7 +311,8 @@ def copy_file_to_testbed(testbed, ssh_config, path, destination): copy_files_to_testbed(testbed, ssh_config, [path], destination) -def copy_files_from_testbed(testbed, ssh_config, paths, destination): +def copy_files_from_testbed(testbed, ssh_config, paths, + destination, sudo=False): """ Copies local files to a remote node. @@ -312,10 +320,15 @@ def copy_files_from_testbed(testbed, ssh_config, paths, destination): @param ssh_config: ssh config of the node @param paths: source paths (remote) as an iterable @param destination: destination folder name (local) + @param sudo: if path to copy requires root access, should be set to true """ if destination is not '' and not destination.endswith('/'): destination = destination + '/' + if sudo: + execute_command(testbed, ssh_config, + 'sudo chmod a+rw %s' % (" ".join(paths))) + if ssh_config.client is None: client, proxy_client = ssh_connect(ssh_config.hostname, ssh_config.port, testbed.username, testbed.password, @@ -341,7 +354,8 @@ def copy_files_from_testbed(testbed, ssh_config, paths, destination): raise SSHException('Failed to copy files from testbed', e) -def copy_file_from_testbed(testbed, ssh_config, path, destination): +def copy_file_from_testbed(testbed, ssh_config, path, + destination, sudo=False): """ Copies a local file to a remote node. @@ -349,8 +363,9 @@ def copy_file_from_testbed(testbed, ssh_config, path, destination): @param ssh_config: ssh config of the node @param path: source path (remote) @param destination: destination folder name (local) + @param sudo: if path to copy requires root access, should be set to true """ - copy_files_from_testbed(testbed, ssh_config, [path], destination) + copy_files_from_testbed(testbed, ssh_config, [path], destination, sudo) def setup_vlans(testbed, node, vlans): diff --git a/rumba/storyboard.py b/rumba/storyboard.py index 26e91fd..ca0cfb5 100644 --- a/rumba/storyboard.py +++ b/rumba/storyboard.py @@ -229,7 +229,7 @@ class Server: def run(self): for node in self.nodes: - logfile = "%s_server.log" % self.ap + logfile = "/tmp/%s_server.log" % self.ap script = r'nohup "$@" > %s 2>&1 & echo "$!"' % (logfile,) run_cmd = self.ap + ( (" " + self.options) if self.options is not None else "" @@ -434,18 +434,21 @@ class StoryBoard: for server in self.servers: server.stop() - def fetch_logs(self, local_dir='.'): + def fetch_logs(self, local_dir=None): + if local_dir is None: + local_dir = self.experiment.log_dir if not os.path.isdir(local_dir): - raise Exception('"%s" is not a directory. Cannot fetch logs.' + raise Exception('Destination "%s" is not a directory. ' + 'Cannot fetch logs.' % local_dir) for node in self.server_nodes: - logs_list = node.execute_command('ls *_server.log') + logs_list = node.execute_command('ls /tmp/*_server.log') logs_list = [x for x in logs_list.split('\n') if x != ''] - logger.info('Log list is:\n%s', logs_list) + logger.debug('Log list is:\n%s', logs_list) node.fetch_files(logs_list, local_dir) for node in self.client_nodes: logs_list = node.execute_command('ls /tmp/*.rumba.log ' '|| echo ""') logs_list = [x for x in logs_list.split('\n') if x != ''] - logger.info('Log list is:\n%s', logs_list) + logger.debug('Log list is:\n%s', logs_list) node.fetch_files(logs_list, local_dir) diff --git a/rumba/utils.py b/rumba/utils.py index 2a8c6b7..78a80aa 100644 --- a/rumba/utils.py +++ b/rumba/utils.py @@ -23,8 +23,8 @@ # License along with this library; if not, write to the Free Software # Foundation, Inc., http://www.fsf.org/about/contact/. # - -import time +import enum +import os import rumba.log as log import rumba.model as model @@ -38,31 +38,133 @@ except NameError: logger = log.get_logger(__name__) -class ExperimentManager(object): +class SwapOutStrategy(enum.Enum): + + NO = 0 + AUTO = 1 + PAUSE = 2 + PROMPT = 3 + + +class SyslogsStrategy(enum.Enum): + + NO = 0 + DEFAULT = 1 + DMESG = 2 + CUSTOM = 3 + + +# Utility names for importing in the scripts +NO_SWAPOUT = SwapOutStrategy.NO +AUTO_SWAPOUT = SwapOutStrategy.AUTO +PAUSE_SWAPOUT = SwapOutStrategy.PAUSE +PROMPT_SWAPOUT = SwapOutStrategy.PROMPT + +NO_SYSLOGS = SyslogsStrategy.NO +DEFAULT_SYSLOGS = SyslogsStrategy.DEFAULT +DMESG_SYSLOGS = SyslogsStrategy.DMESG +CUSTOM_SYSLOGS = SyslogsStrategy.CUSTOM - PROMPT = 1 - AUTO = 2 - NO = 3 - def __init__(self, experiment, do_swap_out=AUTO): +class ExperimentManager(object): + + def __init__(self, + experiment, + swap_out_strategy=AUTO_SWAPOUT, + syslogs_strategy=NO_SYSLOGS, + syslogs=None): assert isinstance(experiment, model.Experiment), \ 'An experiment instance is required.' self.experiment = experiment - self.do_swap_out = do_swap_out + self.swap_out_strategy = swap_out_strategy + self.syslogs_strategy = syslogs_strategy + self.syslogs = [syslogs] if isinstance(syslogs, str) else syslogs + self.use_sudo = self.experiment.testbed.username != 'root' def __enter__(self): pass + def fetch_dmesg_syslog(self, node, node_dir): + node.execute_command('dmesg > /tmp/dmesg') + node.fetch_file('/tmp/dmesg', node_dir) + + def fetch_syslog(self, node, node_dir): + node.fetch_files(self.syslogs, + node_dir, self.use_sudo) + + def fetch_syslogs(self): + local_dir = self.experiment.log_dir + + # Define and set up fetching function + if self.syslogs_strategy == DMESG_SYSLOGS: + fetching_function = self.fetch_dmesg_syslog + elif self.syslogs_strategy == DEFAULT_SYSLOGS: + self.syslogs = self.experiment.testbed.system_logs + fetching_function = self.fetch_syslog + elif self.syslogs_strategy == CUSTOM_SYSLOGS: + assert self.syslogs is not None, \ + 'Custom syslog strategy requires specifying a path' + fetching_function = self.fetch_syslog + else: + raise ValueError('Unknown syslogs strategy %s' + % self.syslogs_strategy) + + for node in self.experiment.nodes: + node_dir = os.path.join(local_dir, node.name) + if not os.path.isdir(node_dir): + os.mkdir(node_dir) + try: + fetching_function(node, node_dir) + except Exception as e: + logger.warning( + 'Could not fetch syslogs of node %s. %s%s', + node.name, + type(e).__name__, + (": " + str(e)) if str(e) != "" else "" + ) + def __exit__(self, exc_type, exc_val, exc_tb): try: - if self.do_swap_out == self.PROMPT: - logger.info('Press ENTER to start swap out.') - input('') - if self.do_swap_out == self.PROMPT \ - or self.do_swap_out == self.AUTO: + # Pause to let the user play + if self.swap_out_strategy == PAUSE_SWAPOUT: + input('Press ENTER to start swap out.') + do_swap_out = True + elif self.swap_out_strategy == PROMPT_SWAPOUT: + do_swap_out = None + while do_swap_out is None: + ans = input('Swap out experiment? (y/n): ') + if ans == 'y': + do_swap_out = True + elif ans == 'n': + do_swap_out = False + else: + print("Only 'y' or 'n' please.") + elif self.swap_out_strategy == AUTO_SWAPOUT: + do_swap_out = True + elif self.swap_out_strategy == NO_SWAPOUT: + do_swap_out = False + else: + logger.warning('Unknown swap-out strategy %s. Swapping out.', + self.swap_out_strategy) + do_swap_out = True + + # Fetch syslogs (if requested) + if self.syslogs_strategy != NO_SYSLOGS: + try: + self.fetch_syslogs() + except Exception as e: + logger.warning( + 'There has been a problem fetching syslogs. %s%s', + type(e).__name__, + ": " + str(e) if str(e) != "" else "" + ) + + # Swap out + if do_swap_out: self.experiment.swap_out() if exc_val is not None: - logger.error('Something went wrong. Got %s: %s', + logger.error('Something went wrong during swap out. ' + 'Got %s: %s', type(exc_val).__name__, str(exc_val)) logger.debug('Exception details:', exc_info=exc_val) finally: @@ -70,4 +172,4 @@ class ExperimentManager(object): # Make sure to print all logs before execution terminates, # Specifically the last two error logs above. return True - # Suppress the exception we logged: no traceback, unless requested. + # Suppress the exception we logged: no traceback, unless logged. @@ -13,7 +13,12 @@ setuptools.setup( license='LGPL', description='Rumba measurement framework for RINA', packages=['rumba', 'rumba.testbeds', 'rumba.prototypes'], - install_requires=['paramiko', 'repoze.lru; python_version<"3.2"', 'contextlib2; python_version<"3.0"'], + install_requires=[ + 'paramiko', + 'repoze.lru; python_version<"3.2"', + 'contextlib2; python_version<"3.0"', + 'enum34; python_version<"3.0"' + ], extras_require={'NumpyAcceleration': ['numpy']}, scripts=['tools/rumba-access'] ) |