From 3081d070cda223afd548645143142e1104b07d83 Mon Sep 17 00:00:00 2001 From: Marco Capitani Date: Fri, 28 Apr 2017 12:10:00 +0200 Subject: model: added policy class + policy-oriented API fixes + Added policy class + Adapted NormalDIF class to use new policy class + NormalDIF constructor argument policy has changed format (parameters dict is now mandatory, added empty dicts where it was not passed) + Added Node.policies field (type: dict[DIF -> Policy]) --- rumba/model.py | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 11 deletions(-) (limited to 'rumba') diff --git a/rumba/model.py b/rumba/model.py index 9187fcb..442933c 100644 --- a/rumba/model.py +++ b/rumba/model.py @@ -112,25 +112,30 @@ class ShimEthDIF(DIF): # Normal DIF # -# @policies [dict] Policies of the normal DIF +# @policies [dict] Policies of the normal DIF. Format: +# dict( componentName: str --> comp_policy: +# dict( policy_name: str --> parameters: +# dict( name: str --> value: str ))) # class NormalDIF(DIF): def __init__(self, name, members=None, policies=None): DIF.__init__(self, name, members) if policies is None: - policies = dict() - self.policies = policies + self.policy = Policy(self) + self.policy = Policy(self, policies=policies) - def add_policy(self, comp, pol): - self.policies[comp] = pol + def add_policy(self, comp, pol, **params): + self.policy.add_policy(comp, pol, **params) - def del_policy(self, comp): - del self.policies[comp] + def del_policy(self, comp=None, policy_name=None): + self.policy.del_policy(comp, policy_name) def show(self): s = DIF.__repr__(self) - for comp, pol in self.policies.items(): - s += "\n Component %s has policy %s" % (comp, pol) + for comp, pol_dict in self.policy.get_policies().items(): + for pol, params in pol_dict.items(): + s += "\n Component %s has policy %s with params %s" \ + % (comp, pol, params) return s @@ -149,10 +154,11 @@ class SSHConfig: # @dif_registrations: Which DIF is registered in which DIF # @registrations: Registrations of names in DIFs # @bindings: Binding of names on the processing system +# @policies: dict of dif -> policy dict to apply for that dif in this node # class Node: def __init__(self, name, difs=None, dif_registrations=None, - registrations=None, bindings=None): + registrations=None, bindings=None, policies=None): self.name = name if difs is None: difs = list() @@ -170,6 +176,13 @@ class Node: self.bindings = bindings self.ssh_config = SSHConfig(name) self.ipcps = [] + if policies is None: + policies = dict() + self.policies = dict() + for dif in self.difs: + if hasattr(dif, 'policy'): + self.policies[dif] = \ + Policy(dif, self, policies.get(dif.name, {})) self._validate() @@ -195,7 +208,7 @@ class Node: for dif in self.registrations[appl]: self._undeclared_dif(dif) - def __repr__(self): + def __repr__(self): # TODO add policies in repr? s = "Node " + self.name + ":\n" s += " DIFs: [ " @@ -240,11 +253,18 @@ class Node: def add_dif(self, dif): self.difs.append(dif) dif.add_member(self) + if hasattr(dif, 'policy'): + self.policies[dif] = Policy(dif, self) self._validate() def del_dif(self, dif): self.difs.remove(dif) dif.del_member(self) + try: + del self.policies[dif] + except KeyError: + # It was not in there, so nothing to do + pass self._validate() def add_dif_registration(self, upper, lower): @@ -271,6 +291,12 @@ class Node: del self.bindings[name] self._validate() + def add_policy(self, dif, component_name, policy_name, **parameters): + self.policies[dif].add_policy(component_name, policy_name, **parameters) + + def del_policy(self, dif, component_name=None, policy_name=None): + self.policies[dif].del_policy(component_name, policy_name) + # Base class representing an IPC Process to be created in the experiment # @@ -319,6 +345,45 @@ class ShimUDPIPCP(IPCP): # TODO: add IP and port +# Class representing DIF and Node policies +class Policy(object): + def __init__(self, dif, node=None, policies=None): + self.dif = dif # type: NormalDIF + self.node = node + if policies is None: + self._dict = dict() + else: + self._dict = policies + + def add_policy(self, component_name, policy_name, **parameters): + self._dict.setdefault(component_name, dict())[policy_name] = parameters + + def get_policies(self, component_name=None, policy_name=None): + if self.node is not None: + policy = self._superimpose(self.dif.policy) + else: + policy = self + if component_name is None: + return policy._dict + elif policy_name is None: + return policy._dict[component_name] + else: + return policy._dict[component_name][policy_name] + + def del_policy(self, component_name=None, policy_name=None): + if component_name is None: + self._dict = dict() + elif policy_name is None: + del self._dict[component_name] + else: + del self._dict[component_name][policy_name] + + def _superimpose(self, other): + base = dict(other._dict) + base.update(self._dict) + return Policy(base) + + # Base class for ARCFIRE experiments # # @name [string] Name of the experiment -- cgit v1.2.3 From e36189f8ba98d8a1b254a8dd300f59d5c12a9430 Mon Sep 17 00:00:00 2001 From: Marco Capitani Date: Tue, 13 Jun 2017 12:44:04 +0200 Subject: IRATI: added per-node policy support, and revamped policy data model. additional: + fixed small bug rlated to vhost checking in qemu. + example scripts now correctly use policies. --- examples/example.py | 6 ++- rumba/model.py | 78 ++++++++++++++++++++++++++++--------- rumba/prototypes/irati.py | 15 ++++--- rumba/prototypes/irati_templates.py | 4 +- rumba/testbeds/qemu.py | 11 +++--- 5 files changed, 81 insertions(+), 33 deletions(-) (limited to 'rumba') diff --git a/examples/example.py b/examples/example.py index 8a68aab..567852f 100755 --- a/examples/example.py +++ b/examples/example.py @@ -20,8 +20,10 @@ import rumba.log as log log.set_logging_level('DEBUG') -n1 = NormalDIF("n1", policies = {"rmt.pff": {"lfa": {}}, - "security-manager": {"passwd": {}}}) +n1 = NormalDIF("n1") + +n1.add_policy("rmt.pff", "lfa") +n1.add_policy("security-manager", "passwd") e1 = ShimEthDIF("e1") diff --git a/rumba/model.py b/rumba/model.py index e0f1dcc..affdcbf 100644 --- a/rumba/model.py +++ b/rumba/model.py @@ -20,9 +20,6 @@ # MA 02110-1301 USA import abc -import random - -import time import rumba.log as log @@ -128,11 +125,11 @@ class ShimEthDIF(DIF): # dict( name: str --> value: str ))) # class NormalDIF(DIF): - def __init__(self, name, members=None, policies=None): + def __init__(self, name, members=None, policy=None): DIF.__init__(self, name, members) - if policies is None: - self.policy = Policy(self) - self.policy = Policy(self, policies=policies) + if policy is None: + policy = Policy(self) + self.policy = policy def add_policy(self, comp, pol, **params): self.policy.add_policy(comp, pol, **params) @@ -162,7 +159,7 @@ class SSHConfig: # # @difs: DIFs the node will have an IPCP in # @dif_registrations: Which DIF is registered in which DIF -# @policies: dict of dif -> policy dict to apply for that dif in this node +# @policies: dict of dif -> policy to apply for that dif in this node # # class Node: @@ -179,13 +176,12 @@ class Node: self.dif_registrations = dif_registrations self.ssh_config = SSHConfig(name) self.ipcps = [] + self.policies = dict() if policies is None: policies = dict() - self.policies = dict() for dif in self.difs: - if hasattr(dif, 'policy'): - self.policies[dif] = \ - Policy(dif, self, policies.get(dif.name, {})) + if hasattr(dif, 'policy'): # check if the dif supports policies + self.policies[dif] = policies.get(dif, Policy(dif, self)) self.client = client self._validate() @@ -208,7 +204,7 @@ class Node: for lower in self.dif_registrations[upper]: self._undeclared_dif(lower) - def __repr__(self): # TODO add policies in repr + def __repr__(self): s = "Node " + self.name + ":\n" s += " DIFs: [ " @@ -226,6 +222,8 @@ class Node: s += ", ".join(rl) s += " ]\n" + s += " Policies: [ " + return s def __hash__(self): @@ -268,6 +266,9 @@ class Node: def del_policy(self, dif, component_name=None, policy_name=None): self.policies[dif].del_policy(component_name, policy_name) + def get_policy(self, dif): + return self.policies[dif] + # Base class representing an IPC Process to be created in the experiment # @@ -317,6 +318,10 @@ class ShimUDPIPCP(IPCP): # Class representing DIF and Node policies +# +# @dif: the dif this policy is applied to. +# @node: the node +# class Policy(object): def __init__(self, dif, node=None, policies=None): self.dif = dif # type: NormalDIF @@ -329,11 +334,11 @@ class Policy(object): def add_policy(self, component_name, policy_name, **parameters): self._dict.setdefault(component_name, dict())[policy_name] = parameters + # + # Fetches effective policy info + # def get_policies(self, component_name=None, policy_name=None): - if self.node is not None: - policy = self._superimpose(self.dif.policy) - else: - policy = self + policy = self._superimpose() if component_name is None: return policy._dict elif policy_name is None: @@ -349,10 +354,45 @@ class Policy(object): else: del self._dict[component_name][policy_name] - def _superimpose(self, other): + # + # Merges this policy into that of its dif, obtaining + # the effective policy acting on self.node. + # + def _superimpose(self): + if self.node is None: + return self + other = self.dif.policy base = dict(other._dict) base.update(self._dict) - return Policy(base) + return Policy(self.dif, self.node, base) + + def __eq__(self, other): + if not isinstance(other, Policy): + return False + else: + return other.dif == self.dif \ + and other.node == self.node \ + and other._dict == self._dict + + def __str__(self): + node_str = (" Node: " + self.node) if self.node is not None else "" + return "Policy[Dif: %(dif)s,%(node_str)s Dict: %(dict)s]" \ + % {"dif": self.dif, "node_str": node_str, "dict": self._dict} + + def __repr__(self): + node_str = (" Node: " + self.node) if self.node is not None else "" + s = "Policy[ Dif: %(dif)s,%(node_str)s" \ + % {"dif": self.dif, "node_str": node_str} + comps = [] + for component in self._dict: + for policy in self._dict[component]: + comps.append("\n Component %s has policy %s with params %s" + % (component, + policy, + self._dict[component][policy])) + s += ",".join(comps) + s += "\n]\n" + return s # Base class for ARCFIRE experiments diff --git a/rumba/prototypes/irati.py b/rumba/prototypes/irati.py index 42afe3b..c01e413 100644 --- a/rumba/prototypes/irati.py +++ b/rumba/prototypes/irati.py @@ -355,11 +355,16 @@ class Experiment(mod.Experiment): "apName": "%s.%s.IPCP" % (dif.name, node_name), "apInstance": "1", "address": 16 + node2id_map[node_name]}) - for path, ps in dif.policies.items(): - # if policy['nodes'] == [] or vmname in policy['nodes']: - # TODO: manage per-node-policies - irati_templates.translate_policy( - difconfs[dif.name][node_name], path, ps, parms=[]) + policy_dict = node.get_policy(dif).get_policies() + for component in policy_dict: + for policy_name in policy_dict[component]: + params = policy_dict[component][policy_name].items() + irati_templates.translate_policy( + difconfs[dif.name][node_name], + component, + policy_name, + params + ) # Dump the DIF Allocator map with open(self.conf_dir('da.map'), 'w') as da_map_file: diff --git a/rumba/prototypes/irati_templates.py b/rumba/prototypes/irati_templates.py index b8d9788..2a5c0f0 100644 --- a/rumba/prototypes/irati_templates.py +++ b/rumba/prototypes/irati_templates.py @@ -238,7 +238,7 @@ def ps_set(d, k, v, parms): if d[k]["name"] == v and "parameters" in d[k]: cur_names = [p["name"] for p in d[k]["parameters"]] for p in parms: - name, value = p.split('=') + name, value = p if name in cur_names: for i in range(len(d[k]["parameters"])): if d[k]["parameters"][i]["name"] == name: @@ -249,7 +249,7 @@ def ps_set(d, k, v, parms): elif len(parms) > 0: d[k]["parameters"] = [ - {'name': p.split('=')[0], 'value': p.split('=')[1]} + {'name': p[0], 'value': p[1]} for p in parms] d[k]["name"] = v diff --git a/rumba/testbeds/qemu.py b/rumba/testbeds/qemu.py index df02ab6..1d449dc 100644 --- a/rumba/testbeds/qemu.py +++ b/rumba/testbeds/qemu.py @@ -111,11 +111,12 @@ class Testbed(mod.Testbed): if os.geteuid() != 0: try: subprocess.check_call(["sudo", "-v"]) - if not os.access("/dev/vhost-net", os.R_OK) \ - or not os.access("/dev/vhost-net", os.W_OK) \ - or not os.access("/dev/kvm", os.R_OK) \ - or not os.access("/dev/kvm", os.W_OK): - raise Exception('Cannot open vhost device. Make sure it is' + if self.vhost and \ + (not os.access("/dev/vhost-net", os.R_OK) + or not os.access("/dev/vhost-net", os.W_OK) + or not os.access("/dev/kvm", os.R_OK) + or not os.access("/dev/kvm", os.W_OK)): + raise Exception('Cannot open vhost device. Make sure it is ' 'available and you have rw permissions ' 'on /dev/vhost-net') except subprocess.CalledProcessError: -- cgit v1.2.3 From d2744740a682701d3f1f714c15fe4265f4e7a495 Mon Sep 17 00:00:00 2001 From: Marco Capitani Date: Thu, 15 Jun 2017 12:54:04 +0200 Subject: ssh_support + log: fixes for python 2 compatibility --- rumba/log.py | 15 +++++++-------- rumba/ssh_support.py | 2 +- 2 files changed, 8 insertions(+), 9 deletions(-) (limited to 'rumba') diff --git a/rumba/log.py b/rumba/log.py index d95c034..bed0170 100644 --- a/rumba/log.py +++ b/rumba/log.py @@ -40,9 +40,8 @@ class RumbaFormatter(logging.Formatter): def __init__(self): super(RumbaFormatter, self).__init__( - fmt='{asctime} | {levelname:3.3} | ' - '{name:11.11} | {message}', - style='{', + fmt='%(asctime)s | %(levelname)3.3s | ' + '%(name)11.11s | %(message)s', datefmt='%H:%M:%S') def format(self, record): @@ -95,11 +94,11 @@ def set_logging_level(level, name=None): """ Set the current logging level to for logger named . If name is not specified, sets the logging level for all rumba loggers. - Accepted levels are: - DEBUG == 10, - INFO == 20, - WARNING == 30, - ERROR == 40, + Accepted levels are: + DEBUG == 10, + INFO == 20, + WARNING == 30, + ERROR == 40, CRITICAL == 50, NOTSET == 0 (resets the logger: its level is set to the default or its parents' level) diff --git a/rumba/ssh_support.py b/rumba/ssh_support.py index a1e1ba4..b0970e1 100644 --- a/rumba/ssh_support.py +++ b/rumba/ssh_support.py @@ -40,7 +40,7 @@ def _print_stream(stream): o_array = o.split('\\n') for oi in o_array: logger.debug(oi) - return o + return o.rstrip() def execute_proxy_commands(testbed, ssh_config, commands, time_out=3): -- cgit v1.2.3 From ac824c787d5805b21616a1d433d4156baccf2a9c Mon Sep 17 00:00:00 2001 From: Marco Capitani Date: Thu, 15 Jun 2017 14:07:19 +0200 Subject: jFed: added backoff; model: fix for rumba-access + added 2 seconds sleep after jfed cli returns to avoid calling not-yet-ready nodes. + moved ssh_info to tools/ folder i.e. where rumba-access assumes it is. --- rumba/model.py | 2 +- rumba/testbeds/jfed.py | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) (limited to 'rumba') diff --git a/rumba/model.py b/rumba/model.py index 9818ac7..352c120 100644 --- a/rumba/model.py +++ b/rumba/model.py @@ -522,7 +522,7 @@ class Experiment: break def dump_ssh_info(self): - f = open('ssh_info', 'w') + f = open('tools/ssh_info', 'w') for node in self.nodes: f.write("%s;%s;%s;%s;%s\n" % (node.name, self.testbed.username, diff --git a/rumba/testbeds/jfed.py b/rumba/testbeds/jfed.py index 83fbce7..e158048 100644 --- a/rumba/testbeds/jfed.py +++ b/rumba/testbeds/jfed.py @@ -22,6 +22,8 @@ import subprocess import getpass import xml.dom.minidom as xml import os.path + +import time import wget import tarfile @@ -173,6 +175,9 @@ class Testbed(mod.Testbed): dir_path = os.path.dirname(os.path.abspath(__file__)) # Complete details of the nodes after swapin + logger.info("Sleeping for two seconds to avoid contacting jfed nodes " + "too soon.") + time.sleep(2) for xml_node in xml_nodes: n_name = xml_node.getAttribute("client_id") intfs = xml_node.getElementsByTagName("interface") @@ -215,8 +220,13 @@ class Testbed(mod.Testbed): if isinstance(ipcp, mod.ShimEthIPCP): if self.if_id[ipcp] == i_name: ipcp.ifname = ifname - logger.debug("Node %s interface %s has name %s." - % (node_n.name, mac, ifname)) + if ifname is None: + logger.error("Could not determine name of node" + "%s interface %s" + % (node_n.name, mac)) + else: + logger.debug("Node %s interface %s has name %s." + % (node_n.name, mac, ifname)) # comp_id = intf.getAttribute("component_id") # comp_arr = comp_id.split(":") # ipcp.ifname = comp_arr[-1] -- cgit v1.2.3 From 232172b651407ad69d593b325f2c003e6666b87f Mon Sep 17 00:00:00 2001 From: Marco Capitani Date: Thu, 15 Jun 2017 14:28:21 +0200 Subject: model:reverted directory change for ssh_info --- rumba/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'rumba') diff --git a/rumba/model.py b/rumba/model.py index 352c120..9818ac7 100644 --- a/rumba/model.py +++ b/rumba/model.py @@ -522,7 +522,7 @@ class Experiment: break def dump_ssh_info(self): - f = open('tools/ssh_info', 'w') + f = open('ssh_info', 'w') for node in self.nodes: f.write("%s;%s;%s;%s;%s\n" % (node.name, self.testbed.username, -- cgit v1.2.3 From 020a36055abb64f611d2a06c006304e663b2661c Mon Sep 17 00:00:00 2001 From: Vincenzo Maffione Date: Thu, 15 Jun 2017 14:56:49 +0200 Subject: prototypes: rlite: use isinstance() rather than type() This is convenient for compatibility between Python 2 and 3. --- rumba/prototypes/rlite.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'rumba') diff --git a/rumba/prototypes/rlite.py b/rumba/prototypes/rlite.py index cc38255..abab080 100644 --- a/rumba/prototypes/rlite.py +++ b/rumba/prototypes/rlite.py @@ -70,11 +70,11 @@ class Experiment(mod.Experiment): for ipcp in node.ipcps: # Generate the command to create the IPCP - if type(ipcp.dif) is mod.NormalDIF: + if isinstance(ipcp.dif, mod.NormalDIF): ipcp_type = 'normal' - elif type(ipcp.dif) is mod.ShimEthDIF: + elif isinstance(ipcp.dif, mod.ShimEthDIF): ipcp_type = 'shim-eth' - elif type(ipcp.dif) is mod.ShimUDPDIF: + elif isinstance(ipcp.dif, mod.ShimUDPDIF): ipcp_type = 'shim-udp4' else: logger.warning( @@ -87,7 +87,7 @@ class Experiment(mod.Experiment): # Generate the command to configure the interface # name for the shim-eth - if type(ipcp.dif) is mod.ShimEthDIF: + if isinstance(ipcp.dif, mod.ShimEthDIF): cmds.append("rlite-ctl ipcp-config %s netdev %s" % (ipcp.name, ipcp.ifname)) -- cgit v1.2.3 From e6574371755b029aaa35c20ea5a0214f75a8a258 Mon Sep 17 00:00:00 2001 From: Sander Vrijders Date: Thu, 15 Jun 2017 15:18:07 +0200 Subject: prototypes: ouroboros: use isinstance() rather than type() This is convenient for compatibility between Python 2 and 3. --- rumba/prototypes/ouroboros.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'rumba') diff --git a/rumba/prototypes/ouroboros.py b/rumba/prototypes/ouroboros.py index 9ac1425..4ad41c6 100644 --- a/rumba/prototypes/ouroboros.py +++ b/rumba/prototypes/ouroboros.py @@ -63,13 +63,13 @@ class Experiment(mod.Experiment): else: cmd = "irm i c n " + ipcp.name - if type(ipcp.dif) is mod.ShimEthDIF: + if isinstance(ipcp.dif, mod.ShimEthDIF): # NOTE: Here to test with fake testbed if ipcp.ifname is None: ipcp.ifname = "eth0" cmd += " type shim-eth-llc if_name " + ipcp.ifname cmd += " dif " + ipcp.dif.name - elif type(ipcp.dif) is mod.NormalDIF: + elif isinstance(ipcp.dif, mod.NormalDIF): cmd += " type normal" if ipcp.dif_bootstrapper: cmd += " dif " + ipcp.dif.name @@ -83,7 +83,7 @@ class Experiment(mod.Experiment): for dif_b in node.dif_registrations[ipcp.dif]: cmd2 += " dif " + dif_b.name cmds2.append(cmd2) - elif type(ipcp.dif) is mod.ShimUDPDIF: + elif isinstance(ipcp.dif, mod.ShimUDPDIF): # FIXME: Will fail, since we don't keep IPs yet cmd += " type shim-udp" cmd += " dif " + ipcp.dif.name -- cgit v1.2.3 From 776d3a91ccf5a93dbd3ecf52278e532d74b38b5e Mon Sep 17 00:00:00 2001 From: Sander Vrijders Date: Thu, 15 Jun 2017 15:21:26 +0200 Subject: rumba: prototypes: Fix bad indentation --- rumba/prototypes/ouroboros.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'rumba') diff --git a/rumba/prototypes/ouroboros.py b/rumba/prototypes/ouroboros.py index 4ad41c6..43cb1d5 100644 --- a/rumba/prototypes/ouroboros.py +++ b/rumba/prototypes/ouroboros.py @@ -63,7 +63,7 @@ class Experiment(mod.Experiment): else: cmd = "irm i c n " + ipcp.name - if isinstance(ipcp.dif, mod.ShimEthDIF): + if isinstance(ipcp.dif, mod.ShimEthDIF): # NOTE: Here to test with fake testbed if ipcp.ifname is None: ipcp.ifname = "eth0" -- cgit v1.2.3 From 67288b3505dab7e9ea74fc87d139301e945b4a78 Mon Sep 17 00:00:00 2001 From: Vincenzo Maffione Date: Thu, 15 Jun 2017 16:30:01 +0200 Subject: irati: update kernel version --- rumba/prototypes/irati_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'rumba') diff --git a/rumba/prototypes/irati_templates.py b/rumba/prototypes/irati_templates.py index 2a5c0f0..7d06c96 100644 --- a/rumba/prototypes/irati_templates.py +++ b/rumba/prototypes/irati_templates.py @@ -11,7 +11,7 @@ ipcmconf_base = { "consoleSocket": "%(varpath)s/var/run/ipcm-console.sock" % env_dict, "pluginsPaths": [ "%(installpath)s/lib/rinad/ipcp" % env_dict, - "/lib/modules/4.1.33-irati/extra" + "/lib/modules/4.9.28-irati/extra" ] }, -- cgit v1.2.3