Merge branch 'master' into master-marco

Updating with ssh changes (and other)

8 files changed, 168 insertions, 55 deletions

diff --git a/rumba/model.py b/rumba/model.py
index 9f6d2c2..5f4f162 100644
--- a/rumba/model.py
+++ b/rumba/model.py
@@ -125,6 +125,13 @@ class NormalDIF(DIF):
             s += "\n       Component %s has policy %s" % (comp, pol)
         return s
 
+# SSH Configuration
+#
+class SSHConfig:
+    def __init__(self, hostname, port=22):
+        self.hostname = hostname
+        self.port = port
+
 # A node in the experiment
 #
 # @difs: DIFs the node will have an IPCP in
@@ -150,7 +157,7 @@ class Node:
         if bindings is None:
             bindings = dict()
         self.bindings = bindings
-        self.full_name = name
+        self.ssh_config = SSHConfig(name)
         self.ipcps = []
 
         self._validate()
@@ -313,10 +320,11 @@ class Experiment:
     @classmethod
     def from_config_file(cls, testbed, filename='demo.conf'):
         """
-        :type testbed: Testbed 
+        :type testbed: Testbed
+        :rtype: Experiment
         :param testbed: the testbed for the experiment
-        :param filename: name of the .conf file 
-        :return: the Experiment
+        :param filename: name of the .conf file
+        :return: an Experiment object
         """
 
         shims = {}
@@ -634,4 +642,3 @@ class Experiment:
         finally:
             # No matter what happens, undo the testbed (testbed-specific)
             self.testbed.swap_out(self)
-
diff --git a/rumba/prototypes/irati.py b/rumba/prototypes/irati.py
index 9c8b004..e06610d 100644
--- a/rumba/prototypes/irati.py
+++ b/rumba/prototypes/irati.py
@@ -42,7 +42,7 @@ class Experiment(mod.Experiment):
         cmds.append("sudo nohup ipcm &> ipcm.log &")
 
         for node in self.nodes:
-            ssh.execute_commands(self.testbed, node.full_name,
+            ssh.execute_commands(self.testbed, node.ssh_config,
                                  cmds, time_out=None)
 
     def run_prototype(self):
diff --git a/rumba/prototypes/ouroboros.py b/rumba/prototypes/ouroboros.py
index 4f78361..4fdacd6 100644
--- a/rumba/prototypes/ouroboros.py
+++ b/rumba/prototypes/ouroboros.py
@@ -20,6 +20,7 @@
 
 import rumba.ssh_support as ssh
 import rumba.model as mod
+import time
 
 # An experiment over the Ouroboros implementation
 class Experiment(mod.Experiment):
@@ -32,11 +33,11 @@ class Experiment(mod.Experiment):
         cmds.append("sudo apt-get update")
         cmds.append("sudo apt-get install cmake protobuf-c-compiler git --yes")
         cmds.append("sudo rm -r ~/ouroboros/build")
-        cmds.append("cd ~/ouroboros; sudo ./install_release.sh")
+        cmds.append("cd ~/ouroboros; sudo ./install_debug.sh /")
         cmds.append("sudo nohup irmd > /dev/null &")
 
         for node in self.nodes:
-            ssh.execute_commands(self.testbed, node.full_name,
+            ssh.execute_commands(self.testbed, node.ssh_config,
                                  cmds, time_out=None)
         return
 
@@ -46,14 +47,99 @@ class Experiment(mod.Experiment):
             for name, ap in node.bindings.items():
                 cmds.append("irm b ap " + ap + " n " + name)
 
-            ssh.execute_commands(self.testbed, node.full_name,
-                                 cmds, time_out=None)
+            ssh.execute_commands(self.testbed, node.ssh_config,
+                                 cmds, time_out = None)
+
+    def reg_names(self):
+        for node in self.nodes:
+            cmds = list()
+            for name, difs in node.registrations.items():
+                cmd = "irm r n " + name
+                for dif in difs:
+                    cmd += " dif " + dif.name
+                cmds.append(cmd)
+
+            ssh.execute_commands(self.testbed, node.ssh_config, cmds,
+                                 time_out = None)
+
+    def create_ipcps(self):
+        for node in self.nodes:
+            cmds = list()
+            for ipcp in node.ipcps:
+                cmds2 = list()
+                if ipcp.dif_bootstrapper == True:
+                    cmd = "irm i b n " + ipcp.name
+                else:
+                    cmd = "irm i c n " + ipcp.name
+
+                if type(ipcp.dif) is mod.ShimEthDIF:
+                    # NOTE: Here to test with fake testbed
+                    if ipcp.ifname is None:
+                        ipcp.ifname = "eth0"
+                    cmd += " type shim-eth-llc if_name " + ipcp.ifname
+                    cmd += " dif " + ipcp.dif.name
+                elif type(ipcp.dif) is mod.NormalDIF:
+                    cmd += " type normal"
+                    if ipcp.dif_bootstrapper == True:
+                        cmd += " dif " + ipcp.dif.name
+                        cmd2 = "irm b i " + ipcp.name + " name " + ipcp.dif.name
+                        cmds2.append(cmd2)
+                        cmd2 = "irm r n " + ipcp.name
+                        for dif_b in node.dif_registrations[ipcp.dif]:
+                            cmd2 += " dif " + dif_b.name
+                        cmds2.append(cmd2)
+                        cmd2 = "irm r n " + ipcp.dif.name
+                        for dif_b in node.dif_registrations[ipcp.dif]:
+                            cmd2 += " dif " + dif_b.name
+                        cmds2.append(cmd2)
+                elif type(ipcp.dif) is mod.ShimUDPDIF:
+                    # FIXME: Will fail, since we don't keep IPs yet
+                    cmd += " type shim-udp"
+                    cmd += " dif " + ipcp.dif.name
+                else:
+                    print("Unsupported IPCP type")
+                    continue
+
+                cmds.append(cmd)
+                for cmd in cmds2:
+                    cmds.append(cmd)
+
+            ssh.execute_commands(self.testbed, node.ssh_config, cmds,
+                                 time_out = None)
+
+    def enroll_ipcps(self):
+        for el in self.enrollments:
+            for e in el:
+                for ipcp in e['enrollee'].ipcps:
+                    if ipcp.dif == e['dif']:
+                        cmds = list()
+                        cmd = "irm i e n " + ipcp.name + " dif " + e['dif'].name
+                        cmds.append(cmd)
+                        cmd = "irm b i " + ipcp.name + " name " + ipcp.dif.name
+                        cmds.append(cmd)
+                        cmd = "irm r n " + ipcp.name
+                        for dif_b in e['enrollee'].dif_registrations[ipcp.dif]:
+                            cmd += " dif " + dif_b.name
+                        cmds.append(cmd)
+                        cmd = "irm r n " + ipcp.dif.name
+                        for dif_b in e['enrollee'].dif_registrations[ipcp.dif]:
+                            cmd += " dif " + dif_b.name
+                        cmds.append(cmd)
+
+                        ssh.execute_commands(self.testbed,
+                                             e['enrollee'].ssh_config,
+                                             cmds, time_out = None)
+                        time.sleep(2)
 
     def run_prototype(self):
-        print("[Ouroboros experiment] start")
-        print("Creating resources...")
         print("Setting up Ouroboros...")
         self.setup_ouroboros()
         print("Binding names...")
         self.bind_names()
-        print("[Ouroboros experiment] end")
+        print("Creating IPCPs")
+        self.create_ipcps()
+        print("Enrolling IPCPs...")
+        self.enroll_ipcps()
+        print("Registering names...")
+        self.reg_names()
+        print("All done, have fun!")
diff --git a/rumba/prototypes/rlite.py b/rumba/prototypes/rlite.py
index f4ff825..0ff680d 100644
--- a/rumba/prototypes/rlite.py
+++ b/rumba/prototypes/rlite.py
@@ -30,22 +30,27 @@ class Experiment(mod.Experiment):
     def setup(self):
         cmds = list()
 
-        cmds.append("sudo apt-get update")
-        cmds.append("sudo apt-get install g++ gcc cmake "
-                    "linux-headers-$(uname -r) "
-                    "protobuf-compiler libprotobuf-dev git --yes")
-        cmds.append("sudo rm -rf ~/rlite")
-        cmds.append("cd ~; git clone https://github.com/vmaffione/rlite")
-        cmds.append("cd ~/rlite && ./configure && make && sudo make install")
-        cmds.append("sudo nohup rlite-uipcps -v DBG -k 0 -U -A &> uipcp.log &")
+        if False: # ubuntu
+            cmds.append("apt-get update")
+            cmds.append("apt-get install g++ gcc cmake "
+                        "linux-headers-$(uname -r) "
+                        "protobuf-compiler libprotobuf-dev git --yes")
+            cmds.append("rm -rf ~/rlite")
+            cmds.append("cd ~; git clone https://github.com/vmaffione/rlite")
+            cmds.append("cd ~/rlite && ./configure && make && sudo make install")
+        cmds.append("modprobe rlite")
+        cmds.append("modprobe rlite-normal")
+        cmds.append("modprobe rlite-shim-eth")
+        cmds.append("modprobe rlite-shim-udp4")
+        cmds.append("modprobe rlite-shim-loopback")
+        cmds.append("nohup rlite-uipcps -v DBG -k 0 -U -A &> uipcp.log &")
 
         for node in self.nodes:
-            ssh.execute_commands(self.testbed, node.full_name,
+            ssh.execute_commands(self.testbed, node.ssh_config,
                                  cmds, time_out=None)
 
     def run_prototype(self):
         print("[RLITE experiment] start")
         print("Setting up rlite on the nodes...")
         self.setup()
-        time.sleep(20)
         print("[RLITE experiment] end")
diff --git a/rumba/ssh_support.py b/rumba/ssh_support.py
index 13ac1c6..ad32d13 100644
--- a/rumba/ssh_support.py
+++ b/rumba/ssh_support.py
@@ -27,13 +27,13 @@ def get_ssh_client():
 
     return ssh_client
 
-def execute_commands(testbed, hostname, commands, time_out=3):
+def execute_commands(testbed, ssh_config, commands, time_out=3):
     '''
     Remote execution of a list of shell command on hostname. By
     default this function will exit (timeout) after 3 seconds.
 
     @param testbed: testbed info
-    @param hostname: host name or ip address of the node
+    @param ssh_config: ssh config of the node
     @param command: *nix shell command
     @param time_out: time_out value in seconds, error will be generated if
     no result received in given number of seconds, the value None can
@@ -42,7 +42,9 @@ def execute_commands(testbed, hostname, commands, time_out=3):
     ssh_client = get_ssh_client()
 
     try:
-        ssh_client.connect(hostname, 22,
+        #print("Connecting to %s@%s:%s (pwd=%s)" % (testbed.username,
+        #        ssh_config.hostname, ssh_config.port, testbed.password))
+        ssh_client.connect(ssh_config.hostname, ssh_config.port,
                            testbed.username, testbed.password,
                            look_for_keys=True, timeout=time_out)
         for command in commands:
@@ -59,13 +61,13 @@ def execute_commands(testbed, hostname, commands, time_out=3):
         print(str(e))
         return
 
-def execute_command(testbed, hostname, command, time_out=3):
+def execute_command(testbed, ssh_config, command, time_out=3):
     '''
     Remote execution of a list of shell command on hostname. By
     default this function will exit (timeout) after 3 seconds.
 
     @param testbed: testbed info
-    @param hostname: host name or ip address of the node
+    @param ssh_config: ssh config of the node
     @param command: *nix shell command
     @param time_out: time_out value in seconds, error will be generated if
     no result received in given number of seconds, the value None can
@@ -76,7 +78,7 @@ def execute_command(testbed, hostname, command, time_out=3):
     ssh_client = get_ssh_client()
 
     try:
-        ssh_client.connect(hostname, 22,
+        ssh_client.connect(ssh_config.hostname, ssh_config.port,
                            testbed.username, testbed.password,
                            look_for_keys=True, timeout=time_out)
         stdin, stdout, stderr = ssh_client.exec_command(command)
@@ -93,20 +95,20 @@ def execute_command(testbed, hostname, command, time_out=3):
         print(str(e))
         return
 
-def copy_file_to_testbed(testbed, hostname, text, file_name):
+def copy_file_to_testbed(testbed, ssh_config, text, file_name):
     '''
     Write a string to a given remote file.
     Overwrite the complete file if it already exists!
 
     @param testbed: testbed info
-    @param hostname: host name or ip address of the node
+    @param ssh_config: ssh config of the node
     @param text: string to be written in file
     @param file_name: file name (including full path) on the host
     '''
     ssh_client = get_ssh_client()
 
     try:
-        ssh_client.connect(hostname, 22,
+        ssh_client.connect(ssh_config.hostname, ssh_config.port,
                            testbed.username,
                            testbed.password,
                            look_for_keys=True)
@@ -140,19 +142,20 @@ def setup_vlan(testbed, node, vlan_id, int_name):
     '''
     print("Setting up VLAN on node " + node.name)
 
+    cmds = list()
     cmd = "sudo ip link add link " + \
           str(int_name) + \
           " name " + str(int_name) + \
           "." + str(vlan_id) + \
           " type vlan id " + str(vlan_id)
-    execute_command(testbed, node.full_name, cmd)
+    cmds.append(cmd)
     cmd = "sudo ifconfig " + \
           str(int_name) + "." + \
           str(vlan_id) + " up"
-    execute_command(node.full_name, cmd, testbed)
+    cmds.append(cmd)
     cmd = "sudo ethtool -K " + \
           str(int_name) + " rxvlan off"
-    execute_command(node.full_name, cmd, testbed)
+    cmds.append(cmd)
     cmd = "sudo ethtool -K " + \
           str(int_name) + " txvlan off"
-    execute_command(node.full_name, cmd, testbed)
+    execute_commands(testbed, node.ssh_config, cmds)
diff --git a/rumba/testbeds/emulab.py b/rumba/testbeds/emulab.py
index c4dbe36..aee438e 100644
--- a/rumba/testbeds/emulab.py
+++ b/rumba/testbeds/emulab.py
@@ -43,6 +43,7 @@ class Testbed(mod.Testbed):
         self.url = url
         self.image = image
         self.ip = dict()
+        self.ops_ssh_config = mod.SSHConfig(self.ops_server())
 
     def ops_server(self):
         '''
@@ -76,7 +77,7 @@ class Testbed(mod.Testbed):
         @return: list of created experiments (strings)
         '''
         cmd = '/usr/testbed/bin/sslxmlrpc_client.py -m experiment getlist'
-        out = ssh.execute_command(self, self.ops_server(), cmd)
+        out = ssh.execute_command(self, self.ops_ssh_config, cmd)
 
         try:
             if project_name != None:
@@ -98,7 +99,7 @@ class Testbed(mod.Testbed):
               self.exp_name + \
               ' direction=in'
 
-        output = ssh.execute_command(self, self.ops_server(), cmd)
+        output = ssh.execute_command(self, self.ops_ssh_config, cmd)
 
         return output
 
@@ -131,8 +132,8 @@ class Testbed(mod.Testbed):
               '" exp="' + exp_name + '" noswapin=true ' + \
               'nsfilepath="' + dest_file_name + '"'
 
-        ssh.execute_command(self, self.ops_server(), cmd, time_out=None)
-        ssh.execute_command(self, self.ops_server(), 'rm ' + dest_file_name)
+        ssh.execute_command(self, self.ops_ssh_config, cmd, time_out=None)
+        ssh.execute_command(self, self.ops_ssh_config, 'rm ' + dest_file_name)
         print("New experiment succesfully created.")
 
     def generate_ns_script(self, experiment):
@@ -182,12 +183,12 @@ class Testbed(mod.Testbed):
               self.exp_name + \
               ' -a | grep State | cut -f2,2 -d " "'
 
-        res = ssh.execute_command(self, self.ops_server(), cmd)
+        res = ssh.execute_command(self, self.ops_ssh_config, cmd)
         active = False
         if res == "active":
             active = True
         while active != True:
-            res = ssh.execute_command(self, self.ops_server(), cmd)
+            res = ssh.execute_command(self, self.ops_ssh_config, cmd)
             if res == "active":
                 active = True
             print("Still waiting")
@@ -201,9 +202,11 @@ class Testbed(mod.Testbed):
         @param experiment: the experiment
         '''
 
-        node_full_name = self.full_name(experiment.nodes[0].name)
+        for node in experiment.nodes:
+            node.ssh_config.hostname = self.full_name(experiment.nodes[0].name)
+
         cmd = 'cat /var/emulab/boot/topomap'
-        topomap = ssh.execute_command(self, node_full_name, cmd)
+        topomap = ssh.execute_command(self, experiment.nodes[0].ssh_config, cmd)
         # Almost as ugly as yo momma
         index = topomap.rfind("# lans")
         topo_array = topomap[:index].split('\\n')[1:-1]
@@ -224,8 +227,7 @@ class Testbed(mod.Testbed):
 
         for node in experiment.nodes:
             cmd = 'cat /var/emulab/boot/ifmap'
-            node_full_name = self.full_name(node.name)
-            output = ssh.execute_command(self, node_full_name, cmd)
+            output = ssh.execute_command(self, node.ssh_config, cmd)
             output = re.split('\\\\n', output)
             for item in output:
                 item = item.split()
@@ -233,7 +235,6 @@ class Testbed(mod.Testbed):
                     if isinstance(ipcp, mod.ShimEthIPCP):
                         if self.ip[ipcp] == item[1]:
                             ipcp.ifname = item[0]
-            node.full_name = self.full_name(node.name)
 
     def swap_in(self, experiment):
         self._create_experiment(experiment)
diff --git a/rumba/testbeds/jfed.py b/rumba/testbeds/jfed.py
index f56e5ba..0dfc904 100644
--- a/rumba/testbeds/jfed.py
+++ b/rumba/testbeds/jfed.py
@@ -111,8 +111,8 @@ class Testbed(mod.Testbed):
 
         for node in experiment.nodes:
             auth_name_r = self.auth_name.replace(".", "-")
-            node.full_name = node.name + "." + self.exp_name + "." + \
-                             auth_name_r + "." + self.auth_name
+            node.ssh_config.hostname = node.name + "." + self.exp_name + "." + \
+                                       auth_name_r + "." + self.auth_name
 
         subprocess.call(["java", "-jar", self.jfed_jar, "create", "-S", \
                          self.proj_name, "--rspec", \
diff --git a/rumba/testbeds/qemu.py b/rumba/testbeds/qemu.py
index 73a4f14..7255b3c 100644
--- a/rumba/testbeds/qemu.py
+++ b/rumba/testbeds/qemu.py
@@ -180,14 +180,14 @@ class Testbed(mod.Testbed):
 
         boot_batch_size = max(1, multiprocessing.cpu_count() // 2)
         booting_budget = boot_batch_size
-        boot_backoff = 12
+        boot_backoff = 12 # in seconds
         base_port = 2222
-        vm_memory = 164
+        vm_memory = 164 # in megabytes
         vm_frontend = 'virtio-net-pci'
 
         vmid = 1
 
-        for node in experiment.nodes:  # type: mod.Node
+        for node in experiment.nodes:
             name = node.name
             vm = self.vms.setdefault(name, {'vm': node, 'ports': []})
             fwdp = base_port + vmid
@@ -195,7 +195,8 @@ class Testbed(mod.Testbed):
             mac = '00:0a:0a:0a:%02x:%02x' % (vmid, 99)
             vm['ssh'] = fwdp
             vm['id'] = vmid
-            node.full_name = "127.0.0.1:%s" % fwdp
+            node.ssh_config.hostname = "localhost"
+            node.ssh_config.port = fwdp
 
             vars_dict = {'fwdp': fwdp, 'id': vmid, 'mac': mac,
                          'bzimage': self.bzimage,
@@ -243,8 +244,10 @@ class Testbed(mod.Testbed):
 
             booting_budget -= 1
             if booting_budget <= 0:
-                print('Sleeping for %s seconds to give '
-                      'the machines time to boot up.' % boot_backoff)
+
+                print('Sleeping %s secs waiting '
+                      'for the VMs to boot' % boot_backoff)
+
                 time.sleep(boot_backoff)
                 booting_budget = boot_batch_size
 
@@ -256,6 +259,14 @@ class Testbed(mod.Testbed):
 
             vmid += 1
 
+        # Wait for the last batch of VMs to start
+        if booting_budget < boot_backoff:
+            tsleep = boot_backoff * (boot_batch_size - booting_budget) / \
+                                            boot_batch_size
+            print('Sleeping %s secs waiting for the last VMs to boot' % tsleep)
+            time.sleep(tsleep)
+
+
     def swap_out(self, experiment):
         """
         :rtype str