IRATI config file generation

4 files changed, 535 insertions, 13 deletions

diff --git a/rumba/model.py b/rumba/model.py
index faf353c..4a9a1be 100644
--- a/rumba/model.py
+++ b/rumba/model.py
@@ -298,7 +298,9 @@ class ShimUDPIPCP(IPCP):
 # @nodes: Nodes in the experiment
 #
 class Experiment:
-    def __init__(self, testbed, nodes=None):
+    def __init__(self, testbed, nodes=None, config_file=None):
+        if config_file:
+            nodes = self.read_config_file(config_file)
         if nodes is None:
             nodes = list()
         self.nodes = nodes
@@ -311,13 +313,12 @@ class Experiment:
         self.generate()
 
     @staticmethod
-    def from_config_file(testbed, filename='demo.conf'):
+    def read_config_file(filename='demo.conf'):
         """
         :type testbed: Testbed 
-        :rtype: Experiment
         :param testbed: the testbed for the experiment
         :param filename: name of the .conf file 
-        :return: an Experiment object
+        :return: the nodes list for the configuration file
         """
 
         shims = {}
@@ -407,7 +408,7 @@ class Experiment:
                                  for x, l in node_data['dif_registrations'].items()}
             parsed_nodes.append(Node(name, difs, dif_registrations))
 
-        return Experiment(testbed=testbed, nodes=parsed_nodes)
+        return parsed_nodes
 
     def __repr__(self):
         s = ""
diff --git a/rumba/prototypes/irati.py b/rumba/prototypes/irati.py
index 37a6fbe..df963c5 100644
--- a/rumba/prototypes/irati.py
+++ b/rumba/prototypes/irati.py
@@ -17,14 +17,17 @@
 # License along with this library; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 # MA  02110-1301  USA
+import copy
+import json
 
 import rumba.ssh_support as ssh
 import rumba.model as mod
+import rumba.prototypes.irati_templates as irati_templates
 
 # An experiment over the IRATI implementation
 class Experiment(mod.Experiment):
-    def __init__(self, testbed, nodes=None):
-        mod.Experiment.__init__(self, testbed, nodes)
+    def __init__(self, testbed, nodes=None, config_file=None):
+        mod.Experiment.__init__(self, testbed, nodes, config_file)
 
     def setup(self):
         cmds = list()
@@ -46,3 +49,164 @@ class Experiment(mod.Experiment):
         print("Setting up IRATI on the nodes...")
         self.setup()
         print("[IRATI experiment] end")
+
+
+class ConfBuilder(object):
+
+    def __init__(self, experiment):
+        self.write_conf(experiment)
+
+    @staticmethod
+    def write_conf(experiment):
+        """
+        :type experiment: Experiment
+        :param experiment: the experiment to be configured
+        """
+        ipcmconfs = dict()
+
+        # TODO ask: what are these, and how are they represented in experiment?
+        # Are these bindings or registration (in node)? In gen.py these are given on a per-dif basis...
+        app_mappings = []
+
+        # If some app directives were specified, use those to build da.map.
+        # Otherwise, assume the standard applications are to be mapped in
+        # the DIF with the highest rank.
+        if len(app_mappings) == 0:
+            if len(experiment.dif_ordering) > 0:
+                for adm in irati_templates.da_map_base["applicationToDIFMappings"]:
+                    adm["difName"] = "%s" % (experiment.dif_ordering[-1],)
+        # else:
+        #     irati_templates.da_map_base["applicationToDIFMappings"] = []
+        #     for apm in app_mappings:
+        #         irati_templates.da_map_base["applicationToDIFMappings"].append({
+        #             "encodedAppName": apm['name'],
+        #             "difName": "%s.DIF" % (apm['dif'])
+        #         })
+
+        # TODO ask: I guess this will need to be added, and in that case we should add it to the qemu plugin too...
+        # Where should we take it in input?
+        manager = False
+        mgmt_dif_name = 'NMS'
+
+        if manager:
+            # Add MAD/Manager configuration
+            irati_templates.ipcmconf_base["addons"] = {
+                "mad": {
+                    "managerAppName": "",
+                    "NMSDIFs": [{"DIF": "%s" % mgmt_dif_name}],
+                    "managerConnections": [{
+                        "managerAppName": "manager-1--",
+                        "DIF": "%s" % mgmt_dif_name
+                    }]
+                }
+            }
+
+        for node in experiment.nodes:  # type: mod.Node
+            ipcmconfs[node.name] = copy.deepcopy(irati_templates.ipcmconf_base)
+            if manager:
+                ipcmconfs[node.name]["addons"]["mad"]["managerAppName"] = "%s.mad-1--" % (node.name)
+
+        difconfs = dict()
+        ipcp2shim_map = {}  # We will need it in a sec
+        for dif in experiment.dif_ordering:  # type: mod.DIF
+            if isinstance(dif, mod.ShimEthDIF):
+                ipcp2shim_map.update({ipcp.name: dif for ipcp in dif.ipcps})
+            elif isinstance(dif, mod.NormalDIF):
+                difconfs[dif.name] = dict()
+                for node in dif.members:
+                    difconfs[dif.name][node.name] = copy.deepcopy(irati_templates.normal_dif_base)
+
+        for node in experiment.nodes:  # type: mod.Node
+            ipcmconf = ipcmconfs[node.name]
+
+            for ipcp in node.ipcps:  # type: mod.ShimEthIPCP
+                if isinstance(ipcp, mod.ShimEthIPCP):
+                    node_name, port_id = ipcp.ifname.split('.')
+                    shim = ipcp2shim_map[ipcp.name]  # type: mod.ShimEthDIF
+                    ipcmconf["ipcProcessesToCreate"].append({
+                        "apName": "eth.%d.IPCP" % int(port_id),
+                        "apInstance": "1",
+                        "difName": shim.name
+                    })
+
+                    template_file_name = 'shimeth.%s.%s.dif' % (node_name, shim.name)
+                    ipcmconf["difConfigurations"].append({
+                        "name": shim.name,
+                        "template": template_file_name
+                    })
+
+                    fout = open(template_file_name, 'w')
+                    fout.write(json.dumps({"difType": "shim-eth-vlan",
+                                           "configParameters": {
+                                               "interface-name": "ifc%d" % (int(port_id),)
+                                           }
+                                           },
+                                          indent=4, sort_keys=True))
+                    fout.close()
+
+        # Run over dif_ordering array, to make sure each IPCM config has
+        # the correct ordering for the ipcProcessesToCreate list of operations.
+        # If we iterated over the difs map, the order would be randomic, and so
+        # some IPCP registrations in lower DIFs may fail. This would happen because
+        # at the moment of registration, it may be that the IPCP of the lower DIF
+        # has not been created yet.
+        shims = ipcp2shim_map.values()
+        for dif in experiment.dif_ordering:  # type: mod.NormalDIF
+
+            if dif in shims:
+                # Shims are managed separately, in the previous loop
+                continue
+
+            for node in dif.members:  # type: mod.Node
+                node_name = node.name
+                node_id = int(node.full_name.split(':')[1]) - 2222
+                ipcmconf = ipcmconfs[node_name]
+
+                # TODO ask: here was vm['id']. Does the name work or does it have to be the id (sequential from 1)?
+                normal_ipcp = {"apName": "%s.%s.IPCP" % (dif.name, node_name),
+                               "apInstance": "1",
+                               "difName": "%s" % (dif.name,),
+                               "difsToRegisterAt": []}
+
+                for lower_dif in node.dif_registrations[dif]:  # type: mod.DIF
+                    normal_ipcp["difsToRegisterAt"].append(lower_dif.name)
+
+                ipcmconf["ipcProcessesToCreate"].append(normal_ipcp)
+
+                ipcmconf["difConfigurations"].append({
+                    "name": "%s" % (dif.name,),
+                    "template": "normal.%s.%s.dif" % (node_name, dif.name,)
+                })
+
+                # Fill in the map of IPCP addresses. This could be moved at difconfs
+                # deepcopy-time
+                # TODO what to do for id? Get it from full_name (i.e.: address:port)? Ugly, but might work
+                for other_node in dif.members:  # type: mod.Node
+                    difconfs[dif.name][other_node.name]["knownIPCProcessAddresses"].append({
+                        "apName": "%s.%s.IPCP" % (dif.name, node_name),
+                        "apInstance": "1",
+                        "address": 16 + node_id
+                    })
+                for path, ps in dif.policies.items():
+                    # if policy['nodes'] == [] or vmname in policy['nodes']:
+                    # TODO: policies can be applied per-node (and not just per-dif)? With what syntax?
+                        irati_templates.translate_policy(difconfs[dif.name][node_name], path,
+                                                         ps, parms=[])
+                    # TODO: what is the syntax for the policy parameters?
+
+        # Dump the DIF Allocator map
+        with open('da.map', 'w') as da_map_file:
+            json.dump(irati_templates.da_map_base, da_map_file, indent=4, sort_keys=True)
+
+        for node in experiment.nodes:
+            # Dump the IPCM configuration files
+            with open('%s.ipcm.conf' % (node.name,), 'w') as node_file:
+                json.dump(ipcmconfs[node.name], node_file, indent=4, sort_keys=True)
+
+        for dif in experiment.dif_ordering:  # type: mod.DIF
+            dif_conf = difconfs.get(dif.name, None)
+            if dif_conf:
+                # Dump the normal DIF configuration files
+                for node in dif.members:
+                    with open('normal.%s.%s.dif' % (node.name, dif.name), 'w') as dif_conf_file:
+                        json.dump(dif_conf[node.name], dif_conf_file, indent=4, sort_keys=True)
diff --git a/rumba/prototypes/irati_templates.py b/rumba/prototypes/irati_templates.py
new file mode 100644
index 0000000..9b03abb
--- /dev/null
+++ b/rumba/prototypes/irati_templates.py
@@ -0,0 +1,349 @@
+#
+# Copyright (C) 2014-2017 Nextworks
+# Author: Vincenzo Maffione <v.maffione@nextworks.it>
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+
+# Environment setup for VMs. Standard linux approach
+env_dict = {'installpath': '/usr', 'varpath': ''}
+
+# Template for a IPCM configuration file
+ipcmconf_base = {
+    "configFileVersion": "1.4.1",
+    "localConfiguration": {
+        "installationPath": "%(installpath)s/bin" % env_dict,
+        "libraryPath": "%(installpath)s/lib" % env_dict,
+        "logPath": "%(varpath)s/var/log" % env_dict,
+        "consoleSocket": "%(varpath)s/var/run/ipcm-console.sock" % env_dict,
+        "pluginsPaths": [
+                "%(installpath)s/lib/rinad/ipcp" % env_dict,
+                "/lib/modules/4.1.33-irati/extra"
+        ]
+        },
+
+    "ipcProcessesToCreate": [],
+    "difConfigurations": [],
+}
+
+
+da_map_base = {
+    "applicationToDIFMappings": [
+        {
+            "encodedAppName": "rina.apps.echotime.server-1--",
+            "difName": "n.DIF"
+        },
+        {
+            "encodedAppName": "traffic.generator.server-1--",
+            "difName": "n.DIF"
+        }
+    ],
+}
+
+
+# Template for a normal DIF configuration file
+normal_dif_base =  {
+    "difType" : "normal-ipc",
+    "dataTransferConstants" : {
+        "addressLength" : 2,
+        "cepIdLength" : 2,
+        "lengthLength" : 2,
+        "portIdLength" : 2,
+        "qosIdLength" : 2,
+        "rateLength" : 4,
+        "frameLength" : 4,
+        "sequenceNumberLength" : 4,
+        "ctrlSequenceNumberLength" : 4,
+        "maxPduSize" : 10000,
+        "maxPduLifetime" : 60000
+    },
+
+    "qosCubes" : [ {
+            "name" : "unreliablewithflowcontrol",
+            "id" : 1,
+            "partialDelivery" : False,
+            "orderedDelivery" : True,
+            "efcpPolicies" : {
+                "dtpPolicySet" : {
+                    "name" : "default",
+                    "version" : "0"
+                },
+                "initialATimer" : 0,
+                "dtcpPresent" : True,
+                "dtcpConfiguration" : {
+                    "dtcpPolicySet" : {
+                        "name" : "default",
+                        "version" : "0"
+                    },
+                    "rtxControl" : False,
+                    "flowControl" : True,
+                    "flowControlConfig" : {
+                        "rateBased" : False,
+                        "windowBased" : True,
+                        "windowBasedConfig" : {
+                            "maxClosedWindowQueueLength" : 10,
+                            "initialCredit" : 200
+                        }
+                    }
+                }
+            }
+        }, {
+            "name" : "reliablewithflowcontrol",
+            "id" : 2,
+            "partialDelivery" : False,
+            "orderedDelivery" : True,
+            "maxAllowableGap": 0,
+            "efcpPolicies" : {
+                "dtpPolicySet" : {
+                    "name" : "default",
+                    "version" : "0"
+                },
+                "initialATimer" : 0,
+                "dtcpPresent" : True,
+                "dtcpConfiguration" : {
+                    "dtcpPolicySet" : {
+                        "name" : "default",
+                        "version" : "0"
+                    },
+                    "rtxControl" : True,
+                    "rtxControlConfig" : {
+                        "dataRxmsNmax" : 5,
+                        "initialRtxTime" : 1000
+                    },
+                    "flowControl" : True,
+                    "flowControlConfig" : {
+                        "rateBased" : False,
+                        "windowBased" : True,
+                        "windowBasedConfig" : {
+                            "maxClosedWindowQueueLength" : 10,
+                            "initialCredit" : 200
+                        }
+                    }
+                }
+            }
+        } ],
+
+    "knownIPCProcessAddresses": [],
+
+    "addressPrefixes" : [ {
+            "addressPrefix" : 0,
+            "organization" : "N.Bourbaki"
+        }, {
+            "addressPrefix" : 16,
+            "organization" : "IRATI"
+        } ],
+
+    "rmtConfiguration" : {
+        "pffConfiguration" : {
+            "policySet" : {
+                "name" : "default",
+                "version" : "0"
+            }
+        },
+        "policySet" : {
+            "name" : "default",
+            "version" : "1"
+        }
+    },
+
+    "enrollmentTaskConfiguration" : {
+        "policySet" : {
+            "name" : "default",
+            "version" : "1",
+            "parameters" : [ {
+                "name"  : "enrollTimeoutInMs",
+                "value" : "10000"
+            }, {
+                "name"  : "watchdogPeriodInMs",
+                "value" : "30000"
+            }, {
+                "name"  : "declaredDeadIntervalInMs",
+                "value" : "120000"
+            }, {
+                "name"  : "neighborsEnrollerPeriodInMs",
+                "value" : "0"
+            }, {
+                "name"  : "maxEnrollmentRetries",
+                "value" : "0"
+            } ]
+        }
+     },
+
+    "flowAllocatorConfiguration" : {
+        "policySet" : {
+            "name" : "default",
+            "version" : "1"
+        }
+    },
+
+    "namespaceManagerConfiguration" : {
+        "policySet" : {
+            "name" : "default",
+            "version" : "1"
+        }
+    },
+
+    "securityManagerConfiguration" : {
+        "policySet" : {
+            "name" : "default",
+            "version" : "1"
+        }
+    },
+
+    "resourceAllocatorConfiguration" : {
+        "pduftgConfiguration" : {
+            "policySet" : {
+                "name" : "default",
+                "version" : "0"
+            }
+        }
+    },
+
+    "routingConfiguration" : {
+        "policySet" : {
+            "name" : "link-state",
+            "version" : "1",
+            "parameters" : [ {
+                    "name"  : "objectMaximumAge",
+                    "value" : "10000"
+                },{
+                    "name"  : "waitUntilReadCDAP",
+                    "value" : "5001"
+                },{
+                    "name"  : "waitUntilError",
+                    "value" : "5001"
+                },{
+                    "name"  : "waitUntilPDUFTComputation",
+                    "value" : "103"
+                },{
+                    "name"  : "waitUntilFSODBPropagation",
+                    "value" : "101"
+                },{
+                    "name"  : "waitUntilAgeIncrement",
+                    "value" : "997"
+                },{
+                    "name"  : "routingAlgorithm",
+                    "value" : "Dijkstra"
+                }]
+        }
+    }
+}
+
+
+def ps_set(d, k, v, parms):
+    if k not in d:
+        d[k] = {'name': '', 'version': '1'}
+
+    if d[k]["name"] == v and "parameters" in d[k]:
+        cur_names = [p["name"] for p in d[k]["parameters"]]
+        for p in parms:
+            name, value = p.split('=')
+            if name in cur_names:
+                for i in range(len(d[k]["parameters"])):
+                    if d[k]["parameters"][i]["name"] == name:
+                        d[k]["parameters"][i]["value"] = value
+                        break
+            else:
+                d[k]["parameters"].append({ 'name': name, 'value': value })
+
+    elif len(parms) > 0:
+        d[k]["parameters"] = [ { 'name': p.split('=')[0], 'value': p.split('=')[1]} for p in parms ]
+
+    d[k]["name"] = v
+
+
+def dtp_ps_set(d, v, parms):
+    for i in range(len(d["qosCubes"])):
+        ps_set(d["qosCubes"][i]["efcpPolicies"], "dtpPolicySet", v, parms)
+
+
+def dtcp_ps_set(d, v, parms):
+    for i in range(len(d["qosCubes"])):
+        ps_set(d["qosCubes"][i]["efcpPolicies"]["dtcpConfiguration"], "dtcpPolicySet", v, parms)
+
+
+policy_translator = {
+    'rmt.pff': lambda d, v, p: ps_set(d["rmtConfiguration"]["pffConfiguration"], "policySet", v, p),
+    'rmt': lambda d, v, p: ps_set(d["rmtConfiguration"], "policySet", v, p),
+    'enrollment-task': lambda d, v, p: ps_set(d["enrollmentTaskConfiguration"], "policySet", v, p),
+    'flow-allocator': lambda d, v, p: ps_set(d["flowAllocatorConfiguration"], "policySet", v, p),
+    'namespace-manager': lambda d, v, p: ps_set(d["namespaceManagerConfiguration"], "policySet", v, p),
+    'security-manager': lambda d, v, p: ps_set(d["securityManagerConfiguration"], "policySet", v, p),
+    'routing': lambda d, v, p: ps_set(d["routingConfiguration"], "policySet", v, p),
+    'resource-allocator.pduftg': lambda d, v, p: ps_set(d["resourceAllocatorConfiguration"], "policySet", v, p),
+    'efcp.*.dtcp': None,
+    'efcp.*.dtp': None,
+}
+
+
+def is_security_path(path):
+    sp = path.split('.')
+    return (len(sp) == 3) and (sp[0] == 'security-manager') and (sp[1] in ['auth', 'encrypt', 'ttl', 'errorcheck'])
+
+
+# Do we know this path ?
+def policy_path_valid(path):
+    if path in policy_translator:
+        return True
+
+    # Try to validate security configuration
+    if is_security_path(path):
+            return True
+
+    return False
+
+
+def translate_security_path(d, path, ps, parms):
+    u1, component, profile = path.split('.')
+    if "authSDUProtProfiles" not in d["securityManagerConfiguration"]:
+        d["securityManagerConfiguration"]["authSDUProtProfiles"] = {}
+    d = d["securityManagerConfiguration"]["authSDUProtProfiles"]
+
+    tr = {'auth': 'authPolicy', 'encrypt': 'encryptPolicy',
+          'ttl': 'TTLPolicy', 'errorcheck': 'ErrorCheckPolicy'}
+
+    if profile == 'default':
+        if profile not in d:
+            d["default"] = {}
+
+        ps_set(d["default"], tr[component], ps, parms)
+
+    else: # profile is the name of a DIF
+        if "specific" not in d:
+            d["specific"] = []
+        j = -1
+        for i in range(len(d["specific"])):
+            if d["specific"][i]["underlyingDIF"] == profile + ".DIF":
+                j = i
+                break
+
+        if j == -1: # We need to create an entry for the new DIF
+            d["specific"].append({"underlyingDIF" : profile + ".DIF"})
+
+        ps_set(d["specific"][j], tr[component], ps, parms)
+
+
+def translate_policy(difconf, path, ps, parms):
+    if path =='efcp.*.dtcp':
+        dtcp_ps_set(difconf, ps, parms)
+
+    elif path == 'efcp.*.dtp':
+        dtp_ps_set(difconf, ps, parms)
+
+    elif is_security_path(path):
+        translate_security_path(difconf, path, ps, parms)
+
+    else:
+        policy_translator[path](difconf, ps, parms)
+
diff --git a/rumba/testbeds/qemu.py b/rumba/testbeds/qemu.py
index 1c5d486..220becc 100644
--- a/rumba/testbeds/qemu.py
+++ b/rumba/testbeds/qemu.py
@@ -26,7 +26,7 @@ import rumba.model as mod
 
 
 class Testbed(mod.Testbed):
-    def __init__(self, exp_name, username, bzimage, initramfs, proj_name="ARCFIRE", password="",
+    def __init__(self, exp_name, bzimage, initramfs, proj_name="ARCFIRE", password="root", username="root",
                  use_vhost=True, qemu_logs_dir=None):
         mod.Testbed.__init__(self, exp_name, username, password, proj_name)
         self.vms = {}
@@ -88,11 +88,12 @@ class Testbed(mod.Testbed):
         e_queue = multiprocessing.Queue()
         print(experiment.dif_ordering)
         for shim in experiment.dif_ordering:
-            command_list = []
             if not isinstance(shim, mod.ShimEthDIF):
                 # Nothing to do here
                 continue
             self.shims.append(shim)
+            ipcps = shim.ipcps
+            command_list = []
             command_list += ('sudo brctl addbr %(br)s\n'
                              'sudo ip link set %(br)s up'
                              % {'br': shim.name}
@@ -123,6 +124,14 @@ class Testbed(mod.Testbed):
                                      ).split('\n')
 
                 vm['ports'].append({'tap_id': tap_id, 'shim': shim, 'port_id': port_id})
+                ipcp_set = [x for x in ipcps if x in node.ipcps]
+                if len(ipcp_set) > 1:
+                    raise Exception("Error: more than one ipcp in common between shim dif %s and node %s"
+                                    % (shim.name, node.name))
+                ipcp = ipcp_set[0]  # type: mod.ShimEthIPCP
+                assert ipcp.name == '%s.%s' % (shim.name, node.name), \
+                    'Incorrect Shim Ipcp found: expected %s.%s, found %s' % (shim.name, node.name, ipcp.name)
+                ipcp.ifname = tap_id
                 # TODO deal with Ip address (shim UDP DIF).
 
             # Avoid stacking processes if one failed before.
@@ -164,14 +173,15 @@ class Testbed(mod.Testbed):
 
         vmid = 1
 
-        for node in experiment.nodes:
-            name = node.full_name
+        for node in experiment.nodes:  # type: mod.Node
+            name = node.name
             vm = self.vms.setdefault(name, {'vm': node, 'ports': []})
             fwdp = base_port + vmid
             fwdc = fwdp + 10000
             mac = '00:0a:0a:0a:%02x:%02x' % (vmid, 99)
             vm['ssh'] = fwdp
             vm['id'] = vmid
+            node.full_name = "127.0.0.1:%s" % fwdp
 
             vars_dict = {'fwdp': fwdp, 'id': vmid, 'mac': mac,
                          'bzimage': self.bzimage,
@@ -197,7 +207,6 @@ class Testbed(mod.Testbed):
                         '-device %(frontend)s,mac=%(mac)s,netdev=mgmt '
                         '-netdev user,id=mgmt,%(hostfwdstr)s '
                         '-vga std '
-                        '-pidfile rina-%(id)s.pid '
                         '-serial file:%(vmname)s.log '
                         % vars_dict
                         )
@@ -225,7 +234,6 @@ class Testbed(mod.Testbed):
             with open('%s/qemu_out_%s' % (self.qemu_logs_dir, vmid), 'w') as out_file:
                 print('DEBUG: executing >> %s' % command)
                 self.boot_processes.append(subprocess.Popen(command.split(), stdout=out_file))
-                pass
 
             vmid += 1