aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xexamples/converged-operator-network.py4
-rwxr-xr-xexamples/scalingtime.py4
-rw-r--r--rumba/model.py12
-rw-r--r--rumba/prototypes/irati.py6
-rw-r--r--rumba/ssh_support.py23
-rw-r--r--rumba/storyboard.py15
-rw-r--r--rumba/utils.py132
-rwxr-xr-xsetup.py7
8 files changed, 167 insertions, 36 deletions
diff --git a/examples/converged-operator-network.py b/examples/converged-operator-network.py
index 6242e31..3b80746 100755
--- a/examples/converged-operator-network.py
+++ b/examples/converged-operator-network.py
@@ -5,7 +5,7 @@
from rumba.model import *
from rumba.storyboard import *
-from rumba.utils import ExperimentManager
+from rumba.utils import *
# import testbed plugins
import rumba.testbeds.emulab as emulab
@@ -186,7 +186,7 @@ exp = rl.Experiment(tb, nodes = [f1n1, f1n2, f1n3, f1n4,
print(exp)
-with ExperimentManager(exp, do_swap_out=ExperimentManager.PROMPT):
+with ExperimentManager(exp, swap_out_strategy=PAUSE_SWAPOUT):
exp.swap_in()
exp.install_prototype()
exp.bootstrap_prototype()
diff --git a/examples/scalingtime.py b/examples/scalingtime.py
index 4a35d67..f0c9285 100755
--- a/examples/scalingtime.py
+++ b/examples/scalingtime.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
from rumba.model import *
-from rumba.utils import ExperimentManager
+from rumba.utils import *
# import testbed plugins
import rumba.testbeds.emulab as emulab
@@ -127,7 +127,7 @@ tb = jfed.Testbed(exp_name = "metro2",
exp = our.Experiment(tb, nodes = nodes)
-with ExperimentManager(exp, do_swap_out = ExperimentManager.PROMPT):
+with ExperimentManager(exp, swap_out_strategy=PAUSE_SWAPOUT):
exp.swap_in()
exp.install_prototype()
exp.bootstrap_prototype()
diff --git a/rumba/model.py b/rumba/model.py
index 5647b1e..2749ae7 100644
--- a/rumba/model.py
+++ b/rumba/model.py
@@ -384,20 +384,22 @@ class Node(object):
destination
)
- def fetch_files(self, paths, destination):
+ def fetch_files(self, paths, destination, sudo=False):
ssh_support.copy_files_from_testbed(
self.ssh_config,
self.ssh_config,
paths,
- destination
+ destination,
+ sudo=sudo
)
- def fetch_file(self, path, destination):
+ def fetch_file(self, path, destination, sudo=False):
ssh_support.copy_file_from_testbed(
self.ssh_config,
self.ssh_config,
path,
- destination
+ destination,
+ sudo=sudo
)
@@ -562,7 +564,7 @@ class Experiment(object):
# If it is None, use /tmp/rumba/{project}
# Wipe it and make it again
exp_name = self.testbed.exp_name.replace('/', '_') # Just in case
- log_dir = '/tmp/rumba/' + exp_name + '/'
+ log_dir = os.path.join(tmp_dir, exp_name)
shutil.rmtree(log_dir, ignore_errors=True)
os.mkdir(log_dir)
self.log_dir = log_dir
diff --git a/rumba/prototypes/irati.py b/rumba/prototypes/irati.py
index 41de9bb..335371e 100644
--- a/rumba/prototypes/irati.py
+++ b/rumba/prototypes/irati.py
@@ -62,7 +62,11 @@ class Experiment(mod.Experiment):
def __init__(self, testbed, nodes=None,
git_repo='https://github.com/IRATI/stack',
git_branch='arcfire', installpath=None, varpath=None):
- mod.Experiment.__init__(self, testbed, nodes, git_repo, git_branch)
+ mod.Experiment.__init__(self,
+ testbed,
+ nodes,
+ git_repo,
+ git_branch)
if installpath is None:
installpath = '/usr'
if varpath is None:
diff --git a/rumba/ssh_support.py b/rumba/ssh_support.py
index 69c049c..98caca2 100644
--- a/rumba/ssh_support.py
+++ b/rumba/ssh_support.py
@@ -37,12 +37,14 @@ try:
except NameError:
pass
+
logger = log.get_logger(__name__)
class SSHException(Exception):
pass
+
def get_ssh_client():
ssh_client = paramiko.SSHClient()
ssh_client.load_system_host_keys()
@@ -64,6 +66,7 @@ def _print_stream(stream):
o_array = []
return '\n'.join(o_array)
+
def ssh_connect(hostname, port, username, password, time_out, proxy_server):
logger.debug('Trying to open a connection towards node %s.' % hostname)
retry = 0
@@ -97,7 +100,7 @@ def ssh_connect(hostname, port, username, password, time_out, proxy_server):
logger.error('If you are sure this is not a man in the ' +
'middle attack, edit that file to remove the ' +
'entry and then hit return to try again.')
- input()
+ input('Key mismatch detected. Press ENTER when ready.')
except (paramiko.ssh_exception.SSHException, EOFError):
retry += 1
logger.error('Failed to connect to host, retrying: ' +
@@ -105,6 +108,7 @@ def ssh_connect(hostname, port, username, password, time_out, proxy_server):
if retry == max_retries:
raise SSHException('Failed to connect to host')
+
def execute_proxy_commands(testbed, ssh_config, commands, time_out=3):
"""
Remote execution of a list of shell command on hostname, using the
@@ -202,6 +206,7 @@ def execute_commands(testbed, ssh_config, commands, time_out=3):
'\n\t'.join(list_print) + '\n')
return o
+
def execute_command(testbed, ssh_config, command, time_out=3):
"""
Remote execution of a list of shell command on hostname. By
@@ -256,6 +261,7 @@ def write_text_to_file(testbed, ssh_config, text, file_name):
except SSHException as e:
raise SSHException('Failed to write text to remote file')
+
def copy_files_to_testbed(testbed, ssh_config, paths, destination):
"""
Copies local files to a remote node.
@@ -292,6 +298,7 @@ def copy_files_to_testbed(testbed, ssh_config, paths, destination):
except Exception as e:
raise SSHException('Failed to copy files to testbed')
+
def copy_file_to_testbed(testbed, ssh_config, path, destination):
"""
Copies a local file to a remote node.
@@ -304,7 +311,8 @@ def copy_file_to_testbed(testbed, ssh_config, path, destination):
copy_files_to_testbed(testbed, ssh_config, [path], destination)
-def copy_files_from_testbed(testbed, ssh_config, paths, destination):
+def copy_files_from_testbed(testbed, ssh_config, paths,
+ destination, sudo=False):
"""
Copies local files to a remote node.
@@ -312,10 +320,15 @@ def copy_files_from_testbed(testbed, ssh_config, paths, destination):
@param ssh_config: ssh config of the node
@param paths: source paths (remote) as an iterable
@param destination: destination folder name (local)
+ @param sudo: if path to copy requires root access, should be set to true
"""
if destination is not '' and not destination.endswith('/'):
destination = destination + '/'
+ if sudo:
+ execute_command(testbed, ssh_config,
+ 'sudo chmod a+rw %s' % (" ".join(paths)))
+
if ssh_config.client is None:
client, proxy_client = ssh_connect(ssh_config.hostname, ssh_config.port,
testbed.username, testbed.password,
@@ -341,7 +354,8 @@ def copy_files_from_testbed(testbed, ssh_config, paths, destination):
raise SSHException('Failed to copy files from testbed', e)
-def copy_file_from_testbed(testbed, ssh_config, path, destination):
+def copy_file_from_testbed(testbed, ssh_config, path,
+ destination, sudo=False):
"""
Copies a local file to a remote node.
@@ -349,8 +363,9 @@ def copy_file_from_testbed(testbed, ssh_config, path, destination):
@param ssh_config: ssh config of the node
@param path: source path (remote)
@param destination: destination folder name (local)
+ @param sudo: if path to copy requires root access, should be set to true
"""
- copy_files_from_testbed(testbed, ssh_config, [path], destination)
+ copy_files_from_testbed(testbed, ssh_config, [path], destination, sudo)
def setup_vlans(testbed, node, vlans):
diff --git a/rumba/storyboard.py b/rumba/storyboard.py
index 26e91fd..ca0cfb5 100644
--- a/rumba/storyboard.py
+++ b/rumba/storyboard.py
@@ -229,7 +229,7 @@ class Server:
def run(self):
for node in self.nodes:
- logfile = "%s_server.log" % self.ap
+ logfile = "/tmp/%s_server.log" % self.ap
script = r'nohup "$@" > %s 2>&1 & echo "$!"' % (logfile,)
run_cmd = self.ap + (
(" " + self.options) if self.options is not None else ""
@@ -434,18 +434,21 @@ class StoryBoard:
for server in self.servers:
server.stop()
- def fetch_logs(self, local_dir='.'):
+ def fetch_logs(self, local_dir=None):
+ if local_dir is None:
+ local_dir = self.experiment.log_dir
if not os.path.isdir(local_dir):
- raise Exception('"%s" is not a directory. Cannot fetch logs.'
+ raise Exception('Destination "%s" is not a directory. '
+ 'Cannot fetch logs.'
% local_dir)
for node in self.server_nodes:
- logs_list = node.execute_command('ls *_server.log')
+ logs_list = node.execute_command('ls /tmp/*_server.log')
logs_list = [x for x in logs_list.split('\n') if x != '']
- logger.info('Log list is:\n%s', logs_list)
+ logger.debug('Log list is:\n%s', logs_list)
node.fetch_files(logs_list, local_dir)
for node in self.client_nodes:
logs_list = node.execute_command('ls /tmp/*.rumba.log '
'|| echo ""')
logs_list = [x for x in logs_list.split('\n') if x != '']
- logger.info('Log list is:\n%s', logs_list)
+ logger.debug('Log list is:\n%s', logs_list)
node.fetch_files(logs_list, local_dir)
diff --git a/rumba/utils.py b/rumba/utils.py
index 2a8c6b7..78a80aa 100644
--- a/rumba/utils.py
+++ b/rumba/utils.py
@@ -23,8 +23,8 @@
# License along with this library; if not, write to the Free Software
# Foundation, Inc., http://www.fsf.org/about/contact/.
#
-
-import time
+import enum
+import os
import rumba.log as log
import rumba.model as model
@@ -38,31 +38,133 @@ except NameError:
logger = log.get_logger(__name__)
-class ExperimentManager(object):
+class SwapOutStrategy(enum.Enum):
+
+ NO = 0
+ AUTO = 1
+ PAUSE = 2
+ PROMPT = 3
+
+
+class SyslogsStrategy(enum.Enum):
+
+ NO = 0
+ DEFAULT = 1
+ DMESG = 2
+ CUSTOM = 3
+
+
+# Utility names for importing in the scripts
+NO_SWAPOUT = SwapOutStrategy.NO
+AUTO_SWAPOUT = SwapOutStrategy.AUTO
+PAUSE_SWAPOUT = SwapOutStrategy.PAUSE
+PROMPT_SWAPOUT = SwapOutStrategy.PROMPT
+
+NO_SYSLOGS = SyslogsStrategy.NO
+DEFAULT_SYSLOGS = SyslogsStrategy.DEFAULT
+DMESG_SYSLOGS = SyslogsStrategy.DMESG
+CUSTOM_SYSLOGS = SyslogsStrategy.CUSTOM
- PROMPT = 1
- AUTO = 2
- NO = 3
- def __init__(self, experiment, do_swap_out=AUTO):
+class ExperimentManager(object):
+
+ def __init__(self,
+ experiment,
+ swap_out_strategy=AUTO_SWAPOUT,
+ syslogs_strategy=NO_SYSLOGS,
+ syslogs=None):
assert isinstance(experiment, model.Experiment), \
'An experiment instance is required.'
self.experiment = experiment
- self.do_swap_out = do_swap_out
+ self.swap_out_strategy = swap_out_strategy
+ self.syslogs_strategy = syslogs_strategy
+ self.syslogs = [syslogs] if isinstance(syslogs, str) else syslogs
+ self.use_sudo = self.experiment.testbed.username != 'root'
def __enter__(self):
pass
+ def fetch_dmesg_syslog(self, node, node_dir):
+ node.execute_command('dmesg > /tmp/dmesg')
+ node.fetch_file('/tmp/dmesg', node_dir)
+
+ def fetch_syslog(self, node, node_dir):
+ node.fetch_files(self.syslogs,
+ node_dir, self.use_sudo)
+
+ def fetch_syslogs(self):
+ local_dir = self.experiment.log_dir
+
+ # Define and set up fetching function
+ if self.syslogs_strategy == DMESG_SYSLOGS:
+ fetching_function = self.fetch_dmesg_syslog
+ elif self.syslogs_strategy == DEFAULT_SYSLOGS:
+ self.syslogs = self.experiment.testbed.system_logs
+ fetching_function = self.fetch_syslog
+ elif self.syslogs_strategy == CUSTOM_SYSLOGS:
+ assert self.syslogs is not None, \
+ 'Custom syslog strategy requires specifying a path'
+ fetching_function = self.fetch_syslog
+ else:
+ raise ValueError('Unknown syslogs strategy %s'
+ % self.syslogs_strategy)
+
+ for node in self.experiment.nodes:
+ node_dir = os.path.join(local_dir, node.name)
+ if not os.path.isdir(node_dir):
+ os.mkdir(node_dir)
+ try:
+ fetching_function(node, node_dir)
+ except Exception as e:
+ logger.warning(
+ 'Could not fetch syslogs of node %s. %s%s',
+ node.name,
+ type(e).__name__,
+ (": " + str(e)) if str(e) != "" else ""
+ )
+
def __exit__(self, exc_type, exc_val, exc_tb):
try:
- if self.do_swap_out == self.PROMPT:
- logger.info('Press ENTER to start swap out.')
- input('')
- if self.do_swap_out == self.PROMPT \
- or self.do_swap_out == self.AUTO:
+ # Pause to let the user play
+ if self.swap_out_strategy == PAUSE_SWAPOUT:
+ input('Press ENTER to start swap out.')
+ do_swap_out = True
+ elif self.swap_out_strategy == PROMPT_SWAPOUT:
+ do_swap_out = None
+ while do_swap_out is None:
+ ans = input('Swap out experiment? (y/n): ')
+ if ans == 'y':
+ do_swap_out = True
+ elif ans == 'n':
+ do_swap_out = False
+ else:
+ print("Only 'y' or 'n' please.")
+ elif self.swap_out_strategy == AUTO_SWAPOUT:
+ do_swap_out = True
+ elif self.swap_out_strategy == NO_SWAPOUT:
+ do_swap_out = False
+ else:
+ logger.warning('Unknown swap-out strategy %s. Swapping out.',
+ self.swap_out_strategy)
+ do_swap_out = True
+
+ # Fetch syslogs (if requested)
+ if self.syslogs_strategy != NO_SYSLOGS:
+ try:
+ self.fetch_syslogs()
+ except Exception as e:
+ logger.warning(
+ 'There has been a problem fetching syslogs. %s%s',
+ type(e).__name__,
+ ": " + str(e) if str(e) != "" else ""
+ )
+
+ # Swap out
+ if do_swap_out:
self.experiment.swap_out()
if exc_val is not None:
- logger.error('Something went wrong. Got %s: %s',
+ logger.error('Something went wrong during swap out. '
+ 'Got %s: %s',
type(exc_val).__name__, str(exc_val))
logger.debug('Exception details:', exc_info=exc_val)
finally:
@@ -70,4 +172,4 @@ class ExperimentManager(object):
# Make sure to print all logs before execution terminates,
# Specifically the last two error logs above.
return True
- # Suppress the exception we logged: no traceback, unless requested.
+ # Suppress the exception we logged: no traceback, unless logged.
diff --git a/setup.py b/setup.py
index c70cf6a..99ff69c 100755
--- a/setup.py
+++ b/setup.py
@@ -13,7 +13,12 @@ setuptools.setup(
license='LGPL',
description='Rumba measurement framework for RINA',
packages=['rumba', 'rumba.testbeds', 'rumba.prototypes'],
- install_requires=['paramiko', 'repoze.lru; python_version<"3.2"', 'contextlib2; python_version<"3.0"'],
+ install_requires=[
+ 'paramiko',
+ 'repoze.lru; python_version<"3.2"',
+ 'contextlib2; python_version<"3.0"',
+ 'enum34; python_version<"3.0"'
+ ],
extras_require={'NumpyAcceleration': ['numpy']},
scripts=['tools/rumba-access']
)