Merge branch 'master' into jfed-irati-fixes

7 files changed, 254 insertions, 38 deletions

diff --git a/README.md b/README.md
index d6e0839..9bd7d38 100644
--- a/README.md
+++ b/README.md
@@ -37,12 +37,19 @@ a look at the examples/ folder.
 
 ## Installation
 
+   For Debian and Ubuntu, the following command will ensure that the
+   required dependencies are installed (replace python-dev with python3-dev
+   if using Python 3):
+
+      # apt-get install build-essential libssl-dev libffi-dev python-dev
+
    Rumba can be found on the
    [PyPi](https://pypi.python.org/pypi/Rumba) and can thus be
    installed through pip, by using `pip install rumba`. However, to
    install the latest version, after cloning the repository, a user
    can also issue `python setup.py install`.
 
+
 ## Supported prototypes
 
  * [IRATI](https://github.com/IRATI/stack) is an open source
@@ -82,16 +89,28 @@ a look at the examples/ folder.
 
    Here the experiment name is rochefort10, the user's name is
    ricksanchez, and the certificate can be found in
-   /home/morty/cert.pem. Please use an absolute path for cert_file for
-   now.
+   /home/morty/cert.pem. An absolute path must be used for cert_file.
 
-   Before running the experiment it is wise to use an SSH agent to
-   avoid having to enter the passphrase for every login to a node by
-   the framework if you are not on an IPv6 enabled network. (Apart
-   from asking for the passphrase to login to the nodes, the framework
+   Before running the rumba you must run an SSH agent in same terminal.
+   This will also avoid you having to enter the passphrase for every
+   login to a node by the framework if you are not on an IPv6 enabled network.
+   (Apart from asking for the passphrase to login to the nodes, the framework
    will always ask for the passphrase since it is needed by the jFed
    CLI as well.) In order to start an SSH agent and to add the
-   certificate, simply perform the following commands:
+   certificate, type the following commands:
+
+       $ eval `ssh-agent`
+       $ ssh-add /home/morty/cert.pem
+
+    To access a node once the experiment swapped in, use the following
+    command (in the same terminal where ssh-agent was run):
+
+        $ ssh -A -oProxyCommand="ssh -i $CERTPATH
+            -o StrictHostKeyChecking=no $USER@bastion.test.iminds.be
+            nc $NODENAME.$EXP.wall2-ilabt-iminds-be.wall2.ilabt.iminds.be 22"
+            $USER@$NODENAME.$EXP.wall2-ilabt-iminds-be.wall2.ilabt.iminds.be
 
-       eval `ssh-agent`
-       ssh-add /home/morty/cert.pem
+    where $CERTPATH is the absolute path of the certificate (e.g.
+    /home/morty/cert.pem), $USER is the jFed username (e.g. "ricksanchez"),
+    $NODENAME is the name of the node you want to access (e.g. "a"),
+    and $EXP is the name of the experiment (e.g. "rochefort10").
diff --git a/examples/example.py b/examples/example.py
index 56193c2..1acc883 100755
--- a/examples/example.py
+++ b/examples/example.py
@@ -17,7 +17,6 @@ import rumba.prototypes.irati as irati
 
 import rumba.log as log
 
-
 log.set_logging_level('DEBUG')
 
 n1 = NormalDIF("n1", policies = {"rmt.pff": "lfa",
@@ -27,19 +26,16 @@ e1 = ShimEthDIF("e1")
 
 a = Node("a",
          difs = [n1, e1],
-         dif_registrations = {n1 : [e1]},
-         registrations = {"a.crap" : [n1]},
-         bindings = {"a.crap" : "/usr/bin/crap"})
+         dif_registrations = {n1 : [e1]})
 
 b = Node("b",
          difs = [e1, n1],
-         dif_registrations = {n1 : [e1]})
+         dif_registrations = {n1 : [e1]},
+         client = True)
 
-tb = qemu.Testbed(exp_name = "example1",
-                  username = "root",
-                  password = "root",
-                  bzimage = '/home/vmaffione/git/rlite/demo/buildroot/bzImage',
-                  initramfs = '/home/vmaffione/git/rlite/demo/buildroot/rootfs.cpio')
+tb = jfed.Testbed(exp_name = "example1",
+                  username = "user1",
+                  cert_file = "/home/user1/cert.pem")
 
 exp = rl.Experiment(tb, nodes = [a, b])
 
@@ -48,5 +44,9 @@ print(exp)
 try:
     exp.swap_in()
     exp.bootstrap_prototype()
+    c1 = Client("rinaperf", options ="-t perf -s 1000 -c 10000")
+    s1 = Server("rinaperf", arrival_rate=2, mean_duration=5, options = "-l", nodes = [a], clients = [c1])
+    sb = StoryBoard(exp, 3600, servers = [s1])
+    sb.start()
 finally:
     exp.swap_out()
diff --git a/examples/jfed-rlite.py b/examples/jfed-rlite.py
new file mode 100755
index 0000000..d80b56e
--- /dev/null
+++ b/examples/jfed-rlite.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+from rumba.model import *
+
+import rumba.testbeds.jfed as jfed
+import rumba.prototypes.rlite as rlite
+
+import rumba.log as log
+
+import argparse
+
+
+description = "Script to run rlite on jfed"
+epilog = "2017 H2020 ARCFIRE"
+
+argparser = argparse.ArgumentParser(description = description,
+                                    epilog = epilog)
+argparser.add_argument('--user', type = str, default = 'vmaffio',
+                       help = "jFed username")
+argparser.add_argument('--cert', type = str,
+                       help = "Absolute path to certificate (.pem) file"
+                            " to be used with jFed",
+                        default = '/home/vmaffione/Downloads/vmaffio-jfed.pem')
+argparser.add_argument('--expname', type = str, default = 'pinocchio',
+                       help = "Name of the experiment within the jFed testbed")
+
+args = argparser.parse_args()
+
+log.set_logging_level('DEBUG')
+
+n1 = NormalDIF("n1")
+
+e1 = ShimEthDIF("e1")
+
+a = Node("a",
+         difs = [n1, e1],
+         dif_registrations = {n1 : [e1]})
+
+b = Node("b",
+         difs = [e1, n1],
+         dif_registrations = {n1 : [e1]})
+
+tb = jfed.Testbed(exp_name = args.expname,
+                  cert_file = args.cert,
+                  username = args.user)
+
+exp = rlite.Experiment(tb, nodes = [a, b])
+
+try:
+    exp.swap_in()
+    exp.install_prototype()
+    exp.bootstrap_prototype()
+finally:
+    exp.swap_out()
diff --git a/rumba/model.py b/rumba/model.py
index 9187fcb..285d937 100644
--- a/rumba/model.py
+++ b/rumba/model.py
@@ -20,6 +20,9 @@
 # MA  02110-1301  USA
 
 import abc
+import random
+
+import time
 
 import rumba.log as log
 
@@ -152,7 +155,7 @@ class SSHConfig:
 #
 class Node:
     def __init__(self, name, difs=None, dif_registrations=None,
-                 registrations=None, bindings=None):
+                 registrations=None, bindings=None, client=False):
         self.name = name
         if difs is None:
             difs = list()
@@ -170,6 +173,7 @@ class Node:
         self.bindings = bindings
         self.ssh_config = SSHConfig(name)
         self.ipcps = []
+        self.client = client
 
         self._validate()
 
@@ -285,7 +289,7 @@ class IPCP:
         self.dif = dif
         self.registrations = []
 
-        # Is this node the first in the DIF, so that it does not need
+        # Is this IPCP the first in its DIF, so that it does not need
         # to enroll to anyone ?
         self.dif_bootstrapper = False
 
@@ -511,6 +515,7 @@ class Experiment:
                 if dif not in node.difs:
                     continue
 
+                # Create an instance of the required IPCP class
                 ipcp = dif.get_ipcp_class()(
                     name='%s.%s' % (dif.name, node.name),
                     node=node, dif=dif)
@@ -519,30 +524,34 @@ class Experiment:
                     for lower in node.dif_registrations[dif]:
                         ipcp.registrations.append(lower)
 
+                node.ipcps.append(ipcp)
+                dif.ipcps.append(ipcp)
+
+    def compute_bootstrappers(self):
+        for node in self.nodes:
+            for ipcp in node.ipcps:
                 ipcp.dif_bootstrapper = True
                 for el in self.enrollments:
                     for e in el:
-                        if e['dif'] != dif:
+                        if e['dif'] != ipcp.dif:
                             # Skip this DIF
                             break
-                        if e['enrollee'] == node:
+                        if e['enrollee'] == ipcp:
                             ipcp.dif_bootstrapper = False
                             # Exit the loops
                             break
                     if not ipcp.dif_bootstrapper:
                         break
 
-                node.ipcps.append(ipcp)
-                dif.ipcps.append(ipcp)
-
-            logger.info("IPCP for node %s: %s", node.name, node.ipcps)
-
     # Examine the nodes and DIFs, compute the registration and enrollment
     # order, the list of IPCPs to create, registrations, ...
     def generate(self):
         self.compute_dif_ordering()
         self.compute_ipcps()
         self.compute_enrollments()
+        self.compute_bootstrappers()
+        for node in self.nodes:
+            logger.info("IPCPs for node %s: %s", node.name, node.ipcps)
 
     @abc.abstractmethod
     def install_prototype(self):
@@ -559,3 +568,124 @@ class Experiment:
     def swap_out(self):
         # Undo the testbed (testbed-specific)
         self.testbed.swap_out(self)
+
+
+# Base class for client programs
+#
+# @ap: Application Process binary
+# @options: Options to pass to the binary
+#
+class Client(object):
+    def __init__(self, ap, options=None):
+        self.ap = ap
+        self.options = options
+
+    def start_process(self, node, duration, start_time):
+        return ClientProcess(self.ap, node, duration, start_time, self.options)
+
+
+# Base class for client processes
+#
+# @ap: Application Process binary
+# @node: The node on which this process should run
+# @duration: The time (in seconds) this process should run
+# @start_time: The time at which this process is started.
+# @options: Options to pass to the binary
+#
+class ClientProcess(Client):
+    def __init__(self, ap, node, duration, start_time, options=None):
+        super(ClientProcess, self).__init__(ap, options=options)
+        self.node = node
+        self.duration = duration
+        self.start_time = start_time
+        self.run()
+        self.running = True
+
+    def run(self):
+        pass  # TODO to be implemented
+
+    def stop(self):
+        pass  # TODO to be implemented
+
+    def check(self, now):
+        if not self.running:
+            return
+        if now - self.start_time >= self.duration:
+            self.stop()
+
+
+# Base class for server programs
+#
+# @ap: Application Process binary
+# @arrival_rate: Average requests/s to be received by this server
+# @mean_duration: Average duration of a client connection (in seconds)
+# @options: Options to pass to the binary
+# @max_clients: Maximum number of clients to serve
+# @clients: Client binaries that will use this server
+# @nodes: Specific nodes to start this server on
+#
+class Server:
+    def __init__(self, ap, arrival_rate, mean_duration,
+                 options=None, max_clients=None,
+                 clients=None, nodes=None):
+        self.ap = ap
+        self.options = options
+        self.max_clients = max_clients
+        if clients is None:
+            clients = list()
+        self.clients = clients
+        self.nodes = nodes
+        self.arrival_rate = arrival_rate  # mean requests/s
+        self.mean_duration = mean_duration  # in seconds
+
+    def add_client(self, client):
+        self.clients.append(client)
+
+    def del_client(self, client):
+        self.clients.remove(client)
+
+    def add_node(self, node):
+        self.nodes.append(node)
+
+    def del_node(self, node):
+        self.nodes.remove(node)
+
+    def get_new_clients(self, interval):
+        """
+        Returns a list of clients of size appropriate to the server's rate.
+         
+        The list's size should be a sample from Poisson(arrival_rate) over
+        interval seconds.
+        Hence, the average size should be interval * arrival_rate.
+        """
+        pass
+
+    def make_client_process(self):
+        """Returns a client of this server"""
+        if len(self.clients) == 0:
+            raise Exception("Server %s has empty client list," % (self,))
+        pass  # TODO should return a ClientProcess
+
+
+# Base class for ARCFIRE storyboards
+#
+# @experiment: Experiment to use as input
+# @duration: Duration of the whole storyboard
+# @servers: App servers available in the network
+#
+class StoryBoard:
+    def __init__(self, experiment, duration, servers=None):
+        self.experiment = experiment
+        self.duration = duration
+        if servers is None:
+            servers = list()
+        self.servers = servers
+
+    def add_server(self, server):
+        self.servers.append(server)
+
+    def del_server(self, server):
+        self.servers.remove(server)
+
+    def start(self):
+        pass
diff --git a/rumba/prototypes/irati.py b/rumba/prototypes/irati.py
index 7004831..b5f8f6b 100644
--- a/rumba/prototypes/irati.py
+++ b/rumba/prototypes/irati.py
@@ -158,12 +158,15 @@ class Experiment(mod.Experiment):
                                   '')
 
         cmds = [self.sudo('hostname %(name)s' % format_args),
+                self.sudo('modprobe rina-irati-core'),
                 self.sudo('chmod a+rw /dev/irati'),
                 self.sudo('mv %(genfilesconf)s /etc' % format_args),
                 self.sudo('mv %(genfilesbin)s /usr/bin') % format_args,
                 self.sudo('chmod a+x /usr/bin/enroll.py') % format_args]
 
         cmds += [self.sudo('modprobe rina-default-plugin'),
+                 self.sudo('modprobe shim-eth-vlan'),
+                 self.sudo('modprobe normal-ipcp'),
                  self.sudo('%(installpath)s/bin/ipcm -a \"%(ipcmcomps)s\" '
                            '-c /etc/%(name)s.ipcm.conf -l %(verb)s &> log &'
                            % format_args)]
diff --git a/rumba/prototypes/rlite.py b/rumba/prototypes/rlite.py
index 8a06b44..625668d 100644
--- a/rumba/prototypes/rlite.py
+++ b/rumba/prototypes/rlite.py
@@ -38,17 +38,21 @@ class Experiment(mod.Experiment):
         ssh.execute_commands(self.testbed, node.ssh_config,
                              cmds, time_out=None)
 
+    # Prepend sudo to all commands if the user is not 'root'
+    def may_sudo(self, cmds):
+        if self.testbed.username != 'root':
+            for i in range(len(cmds)):
+                cmds[i] = "sudo %s" % cmds[i]
+
     def init_nodes(self):
+        # Load kernel modules and start the uipcps daemon
         cmds = ["modprobe rlite",
                 "modprobe rlite-normal",
                 "modprobe rlite-shim-eth",
                 "modprobe rlite-shim-udp4",
                 "modprobe rlite-shim-loopback",
                 "rlite-uipcps -v DBG -k 0 &> uipcp.log &"]
-
-        # Load kernel modules
-
-        # Start the uipcps daemon
+        self.may_sudo(cmds)
 
         for node in self.nodes:
             self.execute_commands(node, cmds)
@@ -80,6 +84,7 @@ class Experiment(mod.Experiment):
                     cmds.append("rlite-ctl ipcp-config %s netdev %s"
                                 % (ipcp.name, ipcp.ifname))
 
+            self.may_sudo(cmds)
             self.execute_commands(node, cmds)
 
     def register_ipcps(self):
@@ -91,6 +96,7 @@ class Experiment(mod.Experiment):
                     cmds.append("rlite-ctl ipcp-register %s %s"
                                 % (ipcp.name, lower.name))
 
+            self.may_sudo(cmds)
             self.execute_commands(node, cmds)
 
     def enroll_ipcps(self):
@@ -103,18 +109,22 @@ class Experiment(mod.Experiment):
                      }
                 cmd = "rlite-ctl ipcp-enroll %(enrollee)s %(dif)s "\
                       "%(lower_dif)s %(enroller)s" % d
-                self.execute_commands(e['enrollee'].node, [cmd])
+                cmds = [cmd]
+                self.may_sudo(cmds)
+                self.execute_commands(e['enrollee'].node, cmds)
                 time.sleep(1)
 
     def install_prototype(self):
         logger.info("installing rlite on all nodes")
-        cmds = ["apt-get update",
-                "apt-get install g++ gcc cmake "
+        cmds = ["sudo apt-get update",
+                "export https_proxy=\"https://proxy.atlantis.ugent.be:8080\"; sudo -E apt-get install g++ gcc cmake "
                 "linux-headers-$(uname -r) "
                 "protobuf-compiler libprotobuf-dev git --yes",
                 "rm -rf ~/rlite",
-                "cd ~; git clone https://github.com/vmaffione/rlite",
-                "cd ~/rlite && ./configure && make && sudo make install"]
+                "cd ~; export https_proxy=\"https://proxy.atlantis.ugent.be:8080\"; git clone https://github.com/vmaffione/rlite",
+                "cd ~/rlite && ./configure && make && sudo make install",
+                "cd ~/rlite && sudo make depmod"
+                ]
 
         for node in self.nodes:
             self.execute_commands(node, cmds)
diff --git a/setup.py b/setup.py
index 92d0c3b..316b6da 100755
--- a/setup.py
+++ b/setup.py
@@ -11,7 +11,7 @@ with open(path.join(here, 'README.md'), encoding='utf-8') as f:
 
 setup(
     name="Rumba",
-    version="0.1",
+    version="0.3",
     url="https://gitlab.com/arcfire/rumba",
     keywords="rina measurement testbed",
     author="Sander Vrijders",