diff options
| -rw-r--r-- | doc/log.rst | 9 | ||||
| -rw-r--r-- | rumba/log.py | 13 | ||||
| -rw-r--r-- | rumba/model.py | 15 | ||||
| -rw-r--r-- | rumba/prototypes/irati.py | 24 | ||||
| -rw-r--r-- | rumba/prototypes/ouroboros.py | 12 | ||||
| -rw-r--r-- | rumba/prototypes/rlite.py | 12 | ||||
| -rw-r--r-- | rumba/storyboard.py | 12 | ||||
| -rw-r--r-- | rumba/testbeds/dockertb.py | 15 | ||||
| -rw-r--r-- | rumba/testbeds/emulab.py | 30 | ||||
| -rw-r--r-- | rumba/testbeds/jfed.py | 45 | ||||
| -rw-r--r-- | rumba/testbeds/local.py | 2 | ||||
| -rw-r--r-- | rumba/testbeds/qemu.py | 32 | ||||
| -rw-r--r-- | rumba/utils.py | 27 |
13 files changed, 190 insertions, 58 deletions
diff --git a/doc/log.rst b/doc/log.rst index 6084cf2..1fd3b6f 100644 --- a/doc/log.rst +++ b/doc/log.rst @@ -1,5 +1,10 @@ Logging ============= -.. automodule:: rumba.log - :member-order: bysource +.. autofunction:: rumba.log.set_logging_level + +.. autofunction:: rumba.log.reset_logging_level + +.. autofunction:: rumba.log.flush_and_kill_logging + +.. autofunction:: rumba.log.flush_log diff --git a/rumba/log.py b/rumba/log.py index 3a21ee1..67070a6 100644 --- a/rumba/log.py +++ b/rumba/log.py @@ -169,7 +169,9 @@ except ImportError: class RumbaFormatter(logging.Formatter): - """The logging.Formatter subclass used by Rumba""" + """ + The logging.Formatter subclass used by Rumba + """ level_name_table = { 'CRITICAL': 'CRT', @@ -232,6 +234,7 @@ def get_logger(name): Returns the logger named <name>. <name> should be the module name, for consistency. If setup has not been called yet, it will call it first. + :param name: the name of the desired logger :return: The logger """ @@ -271,9 +274,8 @@ def set_logging_level(level, name=None): def reset_logging_level(): """ - Resets the current logging levels to the defaults. - Defaults are: rumba -> INFO, - everything else -> ERROR + Resets the current logging levels to the defaults. For Rumba the + default is INFO. """ # Un-sets every logger previously set for logger in loggers_set: @@ -283,6 +285,9 @@ def reset_logging_level(): def flush_log(): + """ + Flush the log. + """ time.sleep(0.1) diff --git a/rumba/model.py b/rumba/model.py index a8d4664..d05e7fe 100644 --- a/rumba/model.py +++ b/rumba/model.py @@ -88,6 +88,11 @@ class Testbed(object): self.system_logs = system_logs def swap_in(self, experiment): + """ + Swaps experiment in + + :param experiment: The experiment. + """ for node in experiment.nodes: node.executor = self.executor @@ -101,8 +106,16 @@ class Testbed(object): def _swap_in(self, experiment): logger.info("_swap_in(): nothing to do") - @abc.abstractmethod def swap_out(self, experiment): + """ + Swaps experiment out + + :param experiment: The experiment. + """ + self._swap_out(experiment) + + @abc.abstractmethod + def _swap_out(self, experiment): logger.info("swap_out(): nothing to do") # Base class for DIFs diff --git a/rumba/prototypes/irati.py b/rumba/prototypes/irati.py index b951d58..4619785 100644 --- a/rumba/prototypes/irati.py +++ b/rumba/prototypes/irati.py @@ -38,8 +38,10 @@ import rumba.log as log logger = log.get_logger(__name__) -# An experiment over the IRATI implementation class Experiment(mod.Experiment): + """ + Represents an IRATI experiment. + """ @staticmethod def make_executor(node, packages, testbed): @@ -62,6 +64,16 @@ class Experiment(mod.Experiment): def __init__(self, testbed, nodes=None, git_repo='https://github.com/IRATI/stack', git_branch='arcfire', installpath=None, varpath=None): + """ + Initializes the experiment class. + + :param testbed: The testbed to run the experiment on. + :param nodes: The list of nodes. + :param git_repo: The git repository to use for installation. + :param git_branch: The branch of the git repository to use. + :param installpath: The installation path of IRATI. + :param varpath: The /var path of IRATI. + """ mod.Experiment.__init__(self, testbed, nodes, @@ -94,7 +106,6 @@ class Experiment(mod.Experiment): return os.path.join(self._conf_dir, path) def install(self): - """Installs IRATI on the nodes.""" packages = ["g++", "gcc", "libtool", "linux-headers-$(uname -r)", "autoconf", "automake", "protobuf-compiler", @@ -117,7 +128,6 @@ class Experiment(mod.Experiment): m_processing.call_in_parallel(names, args, executors) def bootstrap_network(self): - """Creates the network by enrolling and configuring the nodes""" for node in self.nodes: self.process_node(node) self.enroll_nodes() @@ -135,12 +145,6 @@ class Experiment(mod.Experiment): logger.info("IPCPs created and enrolled on all nodes") def process_node(self, node): - """ - Installs the configuration and boots up rina on a node - :type node: mod.Node - :param node: - :return: - """ name = node.name vlans = [] @@ -200,7 +204,6 @@ class Experiment(mod.Experiment): ssh.execute_commands(self.testbed, node.ssh_config, cmds) def enroll_nodes(self): - """Runs the enrollments one by one, respecting dependencies""" logger.info("Starting enrollment phase.") time.sleep(5) for enrollment_list in self.enrollments: @@ -237,7 +240,6 @@ class Experiment(mod.Experiment): return dif.name def write_conf(self): - """Write the configuration files""" # Constants and initializations ipcmconfs = dict() difconfs = dict() diff --git a/rumba/prototypes/ouroboros.py b/rumba/prototypes/ouroboros.py index 97f6e1a..dfbde9b 100644 --- a/rumba/prototypes/ouroboros.py +++ b/rumba/prototypes/ouroboros.py @@ -38,11 +38,21 @@ import rumba.testbeds.dockertb as docker logger = log.get_logger(__name__) -# An experiment over the Ouroboros implementation class Experiment(mod.Experiment): + """ + Represents an Ouroboros experiment. + """ def __init__(self, testbed, nodes=None, git_repo='git://ouroboros.ilabt.imec.be/ouroboros', git_branch='master'): + """ + Initializes the experiment class. + + :param testbed: The testbed to run the experiment on. + :param nodes: The list of nodes. + :param git_repo: The git repository to use for installation. + :param git_branch: The branch of the git repository to use. + """ mod.Experiment.__init__(self, testbed, nodes, git_repo, git_branch) self.r_ipcps = dict() diff --git a/rumba/prototypes/rlite.py b/rumba/prototypes/rlite.py index e1b3745..c056b2f 100644 --- a/rumba/prototypes/rlite.py +++ b/rumba/prototypes/rlite.py @@ -35,12 +35,22 @@ import time logger = log.get_logger(__name__) -# An experiment over the rlite implementation class Experiment(mod.Experiment): + """ + Represents an rlite experiment. + """ def __init__(self, testbed, nodes=None, git_repo='https://github.com/vmaffione/rlite', git_branch='master'): + """ + Initializes the experiment class. + + :param testbed: The testbed to run the experiment on. + :param nodes: The list of nodes. + :param git_repo: The git repository to use for installation. + :param git_branch: The branch of the git repository to use. + """ mod.Experiment.__init__(self, testbed, nodes, git_repo, git_branch, prototype_logs=['/tmp/uipcp.log']) diff --git a/rumba/storyboard.py b/rumba/storyboard.py index ba6883d..2de21ff 100644 --- a/rumba/storyboard.py +++ b/rumba/storyboard.py @@ -48,13 +48,13 @@ except ImportError: logger = log.get_logger(__name__) try: - from numpy.random import poisson - from numpy.random import exponential + from numpy.random import poisson as _poisson + from numpy.random import exponential as _exponential logger.debug("Using numpy for faster and better random variables.") except ImportError: - from rumba.recpoisson import poisson + from rumba.recpoisson import poisson as _poisson - def exponential(mean_duration): + def _exponential(mean_duration): return random.expovariate(1.0 / mean_duration) logger.debug("Falling back to simple implementations.") @@ -249,12 +249,12 @@ class Server(SBEntity): interval seconds. Hence, the average size should be interval * arrival_rate. """ - number = poisson(self.arrival_rate * interval) + number = _poisson(self.arrival_rate * interval) number = int(min(number, self.max_clients)) return [self._make_process_arguments() for _ in range(number)] def get_duration(self): - return exponential(self.actual_parameter) + self.min_duration + return _exponential(self.actual_parameter) + self.min_duration def _make_process_arguments(self, duration=None, node=None, proc_id=None, client=None): diff --git a/rumba/testbeds/dockertb.py b/rumba/testbeds/dockertb.py index d018f8c..6d7c7b5 100644 --- a/rumba/testbeds/dockertb.py +++ b/rumba/testbeds/dockertb.py @@ -36,8 +36,19 @@ from rumba.executors.docker import DockerExecutor logger = log.get_logger(__name__) class Testbed(mod.Testbed): + """ + Represents a docker testbed. + """ def __init__(self, exp_name, base_image, pull_image=True, use_ovs=False): + """ + Initializes the testbed class. + + :param exp_name: The experiment name. + :param base_image: The docker base image. + :param pull_image: Retrieve the docker image from the Docker hub? + :param use_ovs: Use the OVS switch instead of the Linux bridge? + """ mod.Testbed.__init__(self, exp_name, "", "", "") img = base_image.rsplit(":", 1) @@ -159,7 +170,7 @@ class Testbed(mod.Testbed): logger.info("Experiment swapped in") - def swap_out(self, experiment): + def _swap_out(self, experiment): for name, container in self.running_containers.items(): container.remove(force=True) @@ -184,4 +195,4 @@ class Testbed(mod.Testbed): self.running_containers = {} - logger.info("Experiment swapped out")
\ No newline at end of file + logger.info("Experiment swapped out") diff --git a/rumba/testbeds/emulab.py b/rumba/testbeds/emulab.py index 605e326..690612f 100644 --- a/rumba/testbeds/emulab.py +++ b/rumba/testbeds/emulab.py @@ -63,14 +63,14 @@ class Testbed(mod.Testbed): self.url = url self.image = image self.ip = dict() - self.ops_ssh_config = mod.SSHConfig(self.ops_server()) + self.ops_ssh_config = mod.SSHConfig(self._ops_server()) self.executor = SSHExecutor if "wall" in url: self.http_proxy="https://proxy.atlantis.ugent.be:8080" - def ops_server(self): + def _ops_server(self): """ Return server name of the ops-server (is testbed specific). @@ -78,7 +78,7 @@ class Testbed(mod.Testbed): """ return 'ops.' + self.url - def full_name(self, node_name): + def _full_name(self, node_name): """ Return server name of a node. @@ -88,7 +88,7 @@ class Testbed(mod.Testbed): return node_name + '.' + self.exp_name + '.' + \ self.proj_name + '.' + self.url - def get_experiment_list(self, project_name=None): + def _get_experiment_list(self, project_name=None): """ Get list of made emulab experiments accessible with your credentials. @@ -106,7 +106,7 @@ class Testbed(mod.Testbed): except: return {project_name: {project_name: []}} - def swap_exp_in(self): + def _swap_exp_in(self): """ Swaps experiment in. @@ -139,7 +139,7 @@ class Testbed(mod.Testbed): proj_name = self.proj_name exp_name = self.exp_name - exp_list = self.get_experiment_list() + exp_list = self._get_experiment_list() try: if exp_name in exp_list[proj_name][proj_name]: @@ -148,7 +148,7 @@ class Testbed(mod.Testbed): except: logger.info("First experiment to be created for that project.") - ns = self.generate_ns_script(experiment) + ns = self._generate_ns_script(experiment) dest_file_name = '/users/' + self.username + \ '/temp_ns_file.%s.ns' % os.getpid() ssh.write_text_to_file(self, self.ops_ssh_config, ns, dest_file_name) @@ -167,7 +167,7 @@ class Testbed(mod.Testbed): ssh.execute_command(self, self.ops_ssh_config, 'rm ' + dest_file_name) - def generate_ns_script(self, experiment): + def _generate_ns_script(self, experiment): """ Generate ns script based on network graph. Enables to customize default node image. @@ -198,7 +198,7 @@ class Testbed(mod.Testbed): return ns2_script - def wait_until_nodes_up(self): + def _wait_until_nodes_up(self): """ Checks if nodes are up. """ @@ -221,7 +221,7 @@ class Testbed(mod.Testbed): logger.info("Still waiting") time.sleep(5) - def complete_experiment_graph(self, experiment): + def _complete_experiment_graph(self, experiment): """ Gets the interface (ethx) to link mapping. @@ -229,7 +229,7 @@ class Testbed(mod.Testbed): """ for node in experiment.nodes: - node.ssh_config.hostname = self.full_name(node.name) + node.ssh_config.hostname = self._full_name(node.name) node.ssh_config.set_username(self.username) node.ssh_config.set_password(self.password) @@ -273,12 +273,12 @@ class Testbed(mod.Testbed): mod.Testbed.swap_in(self, experiment) self._create_experiment(experiment) - wait = self.swap_exp_in() + wait = self._swap_exp_in() if wait: - self.wait_until_nodes_up() - self.complete_experiment_graph(experiment) + self._wait_until_nodes_up() + self._complete_experiment_graph(experiment) - def swap_out(self, experiment): + def _swap_out(self, experiment): """ Swaps experiment out diff --git a/rumba/testbeds/jfed.py b/rumba/testbeds/jfed.py index a1ceded..24e6737 100644 --- a/rumba/testbeds/jfed.py +++ b/rumba/testbeds/jfed.py @@ -47,11 +47,30 @@ logger = log.get_logger(__name__) class Testbed(mod.Testbed): - + """ + Represents a jFed testbed. + """ def __init__(self, exp_name, username, cert_file, exp_hours="2", proj_name="ARCFIRE", authority="wall2.ilabt.iminds.be", image=None, image_custom=False, image_owner=None, use_physical_machines=None): + """ + Initializes the testbed class. + + :param exp_name: The experiment name. + :param username: User of the experiment. + :param cert_file: Certificate file of the user. + :param exp_hours: Duration of the experiment. + :param proj_name: Project name of the experiment. + :param authority: Actual testbed authority to use. + :param image: Specific image to use. + :param image_custom: Is the image a custom one? + :param image_owner: Creator of the image. + :param use_physical_machines: Try to allocate physical machines. + + .. note:: Supported authorities are wall1.ilabt.iminds.be, + wall2.ilabt.iminds.be, exogeni.net, exogeni.net:umassvmsite. + """ passwd = getpass.getpass(prompt="Password for certificate file: ") mod.Testbed.__init__(self, exp_name, @@ -110,7 +129,13 @@ class Testbed(mod.Testbed): os.remove(tarball) self.flags['no_vlan_offload'] = True - def create_rspec(self, experiment): + def _create_rspec(self, experiment): + """ + Create an rspec which is an XML file with configuration for jFed. + + :param experiment: The experiment. + :return: rspec of the experiment. + """ impl = xml.getDOMImplementation() doc = impl.createDocument(None, "rspec", None) @@ -197,7 +222,12 @@ class Testbed(mod.Testbed): file.write(doc.toprettyxml()) file.close() - def swap_out(self, experiment): + def _swap_out(self, experiment): + """ + Swaps experiment out + + :param experiment: The experiment. + """ try: subprocess.check_call(["java", "-jar", self.jfed_jar, "delete", "-S", self.proj_name, "-s", self.exp_name, @@ -207,9 +237,16 @@ class Testbed(mod.Testbed): raise def _swap_in(self, experiment): + """ + Swaps experiment in + + :param experiment: The experiment. + """ + mod.Testbed.swap_in(self, experiment) + for node in experiment.nodes: node.ssh_config.set_http_proxy(self.http_proxy) - self.create_rspec(experiment) + self._create_rspec(experiment) auth_name_r = self.auth_name.replace(".", "-") diff --git a/rumba/testbeds/local.py b/rumba/testbeds/local.py index 068ee03..f7b505d 100644 --- a/rumba/testbeds/local.py +++ b/rumba/testbeds/local.py @@ -61,7 +61,7 @@ class Testbed(mod.Testbed): logger.info("Experiment swapped in") - def swap_out(self, experiment): + def _swap_out(self, experiment): """ Does not actually swap the experiment out. diff --git a/rumba/testbeds/qemu.py b/rumba/testbeds/qemu.py index 5636e75..218f3e1 100644 --- a/rumba/testbeds/qemu.py +++ b/rumba/testbeds/qemu.py @@ -46,9 +46,27 @@ logger = log.get_logger(__name__) class Testbed(mod.Testbed): + """ + Represents a QEMU testbed. + """ def __init__(self, exp_name, bzimage_path=None, initramfs_path=None, proj_name="ARCFIRE", password="root", username="root", use_vhost=True): + """ + Initializes the testbed class. + + :param exp_name: The experiment name. + :param bzimage_path: Path of the bzimage. + :param initramfs_path: Path of the initramfs. + :param proj_name: Project name of the experiment. + :param password: Password of the user. + :param username: User of the VM. + :param use_vhost: Use virtual hosting or not? + + .. note:: In case no bzimage or initramfs is provided, Rumba + will automatically download the latest version available + from a repository. + """ mod.Testbed.__init__(self, exp_name, username, password, proj_name, system_logs=['/var/log/messages']) @@ -63,7 +81,7 @@ class Testbed(mod.Testbed): self.executor = SSHExecutor(self) # Prepend sudo to all commands if the user is not 'root' - def may_sudo(self, cmds): + def _may_sudo(self, cmds): if os.geteuid() != 0: for i, cmd in enumerate(cmds): cmds[i] = "sudo %s" % cmd @@ -106,7 +124,7 @@ class Testbed(mod.Testbed): else: results_queue.put("Command chain ran with %d error(s)" % errors) - def recover_if_names(self, experiment): + def _recover_if_names(self, experiment): for node in experiment.nodes: for ipcp in node.ipcps: if isinstance(ipcp, mod.ShimEthIPCP): @@ -228,7 +246,7 @@ class Testbed(mod.Testbed): executors = [] for i, script in enumerate(br_tab_scripts): names.append(i) - self.may_sudo(script) + self._may_sudo(script) args.append(script) executors.append(executor) @@ -335,7 +353,7 @@ class Testbed(mod.Testbed): logger.info('All VMs are running. Moving on...') - self.recover_if_names(experiment) + self._recover_if_names(experiment) for node in experiment.nodes: pub_key = os.path.join(os.path.expanduser("~"), @@ -349,7 +367,7 @@ class Testbed(mod.Testbed): logger.info('Experiment has been successfully swapped in.') - def swap_out(self, experiment): + def _swap_out(self, experiment): """ :rtype str :return: The script to tear down the experiment @@ -389,7 +407,7 @@ class Testbed(mod.Testbed): 'ip tuntap del mode tap name %(tap)s' % {'tap': tap, 'br': shim.name} ).split('\n') - self.may_sudo(commands) + self._may_sudo(commands) names.append(index) index += 1 @@ -412,7 +430,7 @@ class Testbed(mod.Testbed): 'brctl delbr %(br)s' % {'br': shim.name} ).split('\n') - self.may_sudo(commands) + self._may_sudo(commands) names.append(index) index += 1 diff --git a/rumba/utils.py b/rumba/utils.py index a522e94..f6d6c1c 100644 --- a/rumba/utils.py +++ b/rumba/utils.py @@ -47,7 +47,9 @@ logger = log.get_logger(__name__) class SwapOutStrategy(enum.Enum): - + """ + What action to perform on swap-out. + """ NO = 0 AUTO = 1 PAUSE = 2 @@ -55,7 +57,9 @@ class SwapOutStrategy(enum.Enum): class SyslogsStrategy(enum.Enum): - + """ + What prototype and system logs to retrieve. + """ NO = 0 DEFAULT = 1 DMESG = 2 @@ -75,12 +79,29 @@ CUSTOM_SYSLOGS = SyslogsStrategy.CUSTOM class ExperimentManager(object): - + """ + Helper class for running a Rumba experiment. + """ def __init__(self, experiment, swap_out_strategy=AUTO_SWAPOUT, syslogs_strategy=NO_SYSLOGS, syslogs=None): + """ + Initializes the ExperimentManager. + + :param experiment: The experiment name. + :param swap_out_strategy: What action to perform on swap-out. + :param syslog_strategy: What system and prototype logs to retrieve + before swap-out. + :param syslogs: The location of the syslogs in case of custom syslogs. + + .. note:: Options for swap_out_strategy are NO_SWAPOUT, AUTO_SWAPOUT, + PAUSE_SWAPOUT, PROMPT_SWAPOUT. + + .. note:: Options for syslog_strategy are NO_SYSLOGS, DEFAULT_SYSLOGS, + DMESG_SYSLOGS, CUSTOM_SYSLOGS. + """ assert isinstance(experiment, model.Experiment), \ 'An experiment instance is required.' self.experiment = experiment |