rumba: Add visualisation tool

The draw_experiment(exp) function will draw a 3D representation of the
network. Depends on igraph and plotly packages.

Signed-off-by: Dimitri Staessens <dimitri@ouroboros.rocks>

3 files changed, 496 insertions, 47 deletions

diff --git a/examples/scalingtime.py b/examples/scalingtime.py
index bf4a739..ff3c2ed 100755
--- a/examples/scalingtime.py
+++ b/examples/scalingtime.py
@@ -16,20 +16,20 @@ import rumba.prototypes.rlite as rl
 import rumba.prototypes.irati as irati
 
 import argparse
-import sys
 
+from rumba.visualizer import draw_experiment
 
 description = "Script to create the demo from the Rumba paper"
 
-argparser = argparse.ArgumentParser(description = description)
+argparser = argparse.ArgumentParser(description=description)
 
 # Parameters to set the experiment size
-argparser.add_argument('--metro-networks', type = int, default = '2',
-                       help = "Number of metro networks")
-argparser.add_argument('--metro-nodes', type = int, default = '5',
-                       help = "Number of nodes per metro")
-argparser.add_argument('--end-nodes', type = int, default = '2',
-                       help = "Number of nodes connected to each metro")
+argparser.add_argument('--metro-networks', type=int, default='2',
+                       help="Number of metro networks")
+argparser.add_argument('--metro-nodes', type=int, default='5',
+                       help="Number of nodes per metro")
+argparser.add_argument('--end-nodes', type=int, default='2',
+                       help="Number of nodes connected to each metro")
 
 # Other parameters
 argparser.add_argument('--verbosity', metavar='VERBOSITY', type=str,
@@ -57,8 +57,8 @@ qemu_p.add_argument('--use_vhost', action='store_true',
                     default=False, help='Use vhost')
 qemu_p.add_argument('--qemu_logs_dir', metavar='QEMU_LOGS', type=str,
                     default=None, help='path to the folder for qemu logs')
-qemu_p.add_argument('--username', type = str, help = "Username")
-qemu_p.add_argument('--password', type = str, help = "Password")
+qemu_p.add_argument('--username', type=str, help="Username")
+qemu_p.add_argument('--password', type=str, help="Password")
 
 emulab_p.add_argument('--url', metavar='URL', type=str,
                       default="wall2.ilabt.iminds.be",
@@ -66,27 +66,27 @@ emulab_p.add_argument('--url', metavar='URL', type=str,
 emulab_p.add_argument('--image', metavar='IMG', type=str,
                       default="UBUNTU14-64-STD",
                       help='Ubuntu image')
-emulab_p.add_argument('--username', type = str, required=True,
-                      help = "Username")
-emulab_p.add_argument('--password', type = str, help = "Password")
-emulab_p.add_argument('--exp-name', type = str, required=True,
-                      help = "Name of the experiment")
+emulab_p.add_argument('--username', type=str, required=True,
+                      help="Username")
+emulab_p.add_argument('--password', type=str, help="Password")
+emulab_p.add_argument('--exp-name', type=str, required=True,
+                      help="Name of the experiment")
 
-jfed_p.add_argument('--cert-file', type = str,
+jfed_p.add_argument('--cert-file', type=str,
                     required=True,
-                    help = "Absolute path to certificate (.pem) file"
+                    help="Absolute path to certificate (.pem) file"
                     " to be used with jFed")
-jfed_p.add_argument('--authority', type = str, default = 'exogeni.net',
-                    help = "jFed testbed to use")
+jfed_p.add_argument('--authority', type=str, default='exogeni.net',
+                    help="jFed testbed to use")
 jfed_p.add_argument('--image', metavar='IMAGE', type=str,
                     default=None,
                     help='Image to be used')
 jfed_p.add_argument('--exp-hours', metavar='HOURS', type=str,
                     default="2", help='Experiment hours')
-jfed_p.add_argument('--username', type = str, required=True,
-                    help = "Username")
-jfed_p.add_argument('--exp-name', type = str, required=True,
-                    help = "Name of the experiment")
+jfed_p.add_argument('--username', type=str, required=True,
+                    help="Username")
+jfed_p.add_argument('--exp-name', type=str, required=True,
+                    help="Name of the experiment")
 
 docker_p.add_argument('--base-image', metavar='DOCKIMAGE', type=str,
                       default=None, required=True,
@@ -117,19 +117,19 @@ for i in range(0, args.metro_networks):
             e = ShimEthDIF("e-" + str(i) + "-" + str(j))
         else:
             e = None
-        if e_prev == None and e != None:
+        if e_prev is None and e is not None:
             node = Node("metro-" + str(i) + "-" + str(j),
-                        difs = [m, pi, e],
-                        dif_registrations = {pi : [m], m : [e]})
+                        difs=[m, pi, e],
+                        dif_registrations={pi: [m], m: [e]})
             m1 = node
-        elif e_prev != None and e != None:
+        elif e_prev is not None and e is not None:
             node = Node("metro-" + str(i) + "-" + str(j),
-                        difs = [m, pi, e, e_prev],
-                        dif_registrations = {pi : [m], m : [e, e_prev]})
+                        difs=[m, pi, e, e_prev],
+                        dif_registrations={pi: [m], m: [e, e_prev]})
         else:
             node = Node("metro-" + str(i) + "-" + str(j),
-                        difs = [m, pi, e_prev],
-                        dif_registrations = {pi : [m], m : [e_prev]})
+                        difs=[m, pi, e_prev],
+                        dif_registrations={pi: [m], m: [e_prev]})
             mn = node
         nodes.append(node)
         e_prev = e
@@ -137,8 +137,8 @@ for i in range(0, args.metro_networks):
         for k in range(0, args.end_nodes):
             ee = ShimEthDIF("e-" + str(i) + "-" + str(j) + "-" + str(k))
             end_node = Node("end-" + str(i) + "-" + str(j) + "-" + str(k),
-                            difs = [pi, ee],
-                            dif_registrations = {pi : [ee]})
+                            difs=[pi, ee],
+                            dif_registrations={pi: [ee]})
             node.add_dif(ee)
             node.add_dif_registration(pi, ee)
             nodes.append(end_node)
@@ -155,12 +155,12 @@ for i in range(0, args.metro_networks):
     e2 = ShimEthDIF("c-e-" + str(i) + "-2")
 
     c1 = Node("c-" + str(i) + "-0",
-              difs = [c, m, pi, e0, e2],
-              dif_registrations = {pi : [m, c], m : [e0], c : [e2]})
+              difs=[c, m, pi, e0, e2],
+              dif_registrations={pi: [m, c], m: [e0], c: [e2]})
 
     c2 = Node("c-" + str(i) + "-1",
-              difs = [c, m, pi, e1, e2],
-              dif_registrations = {pi : [m, c], m : [e1], c : [e2]})
+              difs=[c, m, pi, e1, e2],
+              dif_registrations={pi: [m, c], m: [e1], c: [e2]})
 
     nodes.append(c1)
     nodes.append(c2)
@@ -235,12 +235,14 @@ else:
     print(argparser.format_help())
     exit(1)
 
-exp = exp_class(testbed, nodes = nodes)
-
-with ExperimentManager(exp, swap_out_strategy=PAUSE_SWAPOUT):
-    exp.swap_in()
-    if not isinstance(testbed, docker.Testbed) \
-       and not isinstance(testbed, qemu.Testbed) \
-       and not isinstance(testbed, local.Testbed):
-        exp.install_prototype()
-    exp.bootstrap_prototype()
+exp = exp_class(testbed, nodes=nodes)
+
+if __name__ == "__main__":
+    draw_experiment(exp)
+    with ExperimentManager(exp, swap_out_strategy=PROMPT_SWAPOUT):
+        exp.swap_in()
+        if not isinstance(testbed, docker.Testbed) \
+                and not isinstance(testbed, qemu.Testbed) \
+                and not isinstance(testbed, local.Testbed):
+            exp.install_prototype()
+        exp.bootstrap_prototype()
diff --git a/rumba/prototypes/ouroboros.py b/rumba/prototypes/ouroboros.py
index ed79b02..9cdd3c9 100644
--- a/rumba/prototypes/ouroboros.py
+++ b/rumba/prototypes/ouroboros.py
@@ -469,7 +469,7 @@ class Experiment(mod.Experiment):
             cmds.append('rm -rf /tmp/ouroboros || true')
             cmds.append('kill -9 $(ps axjf | grep \'sudo irmd\' | grep -v grep | cut -f4 -d " ") || true')
         else:
-            cmds.append('killall -15 irmd || true')
+            cmds.append('killall -15 irmd')
 
         logger.info("Killing Ouroboros...")
         if isinstance(self.testbed, local.Testbed):
diff --git a/rumba/visualizer.py b/rumba/visualizer.py
new file mode 100755
index 0000000..f8c18e6
--- /dev/null
+++ b/rumba/visualizer.py
@@ -0,0 +1,447 @@
+#!/usr/bin/python
+
+import igraph as ig
+import plotly.graph_objs as go
+
+from math import sin, cos, pi, sqrt
+
+__all__ = ['draw_network', 'draw_experiment', 'get_default_test_network']
+
+import rumba.elements.topology
+
+type_to_marker = {
+    'eth-dix': 'diamond',
+    'eth-llc': 'diamond',
+    'eth-udp': 'diamond',
+    'unicast': 'circle',
+    'broadcast': 'square'
+}
+
+
+def rumba_to_type(_type):
+    if _type == rumba.elements.topology.ShimEthIPCP:
+        return 'eth-dix'
+    if _type == rumba.elements.topology.ShimUDPIPCP:
+        return 'eth-udp'
+    if _type == rumba.elements.topology.IPCP:
+        return 'unicast'
+
+
+def get_default_test_network():
+    return {
+        'nodes': list(range(8)),
+        'layers': {
+            'bottom_layer1': {
+                'type': 'eth-dix',
+                'nodes': list(range(5)),
+                'edges': [(0, 1), (0, 2), (0, 3), (1, 3), (2, 3), (1, 4), (2, 4)]
+            },
+            'bottom_layer2': {
+                'type': 'eth-llc',
+                'nodes': list(range(4, 8)),
+                'edges': [(4, 5), (5, 6), (6, 7), (4, 7)]
+            },
+            'medium_layer': {
+                'type': 'unicast',
+                'nodes': [0, 2, 4, 5, 7],
+                'edges': [(0, 2), (0, 4), (4, 5), (5, 7), (2, 5), (4, 7)]
+            },
+            'top_layer': {
+                'type': 'unicast',
+                'nodes': [0, 7],
+                'edges': [(0, 7)]
+            }
+        },
+        'registrations': {
+            'medium_layer': {
+                'bottom_layer1': [0, 2, 4],
+                'bottom_layer2': [4, 5, 7]
+            },
+            'top_layer': {
+                'medium_layer': [0, 7]
+            }
+        }
+    }
+
+
+def _get_nodes_in_dif(exp, dif):
+    nodes = []
+    n = 0
+    for node in exp.nodes:
+        if dif in [d.name for d in node.difs]:
+            nodes.append(n)
+        n += 1
+
+    return nodes
+
+
+def _get_node_index(exp, node):
+    n = 0
+    for _node in exp.nodes:
+        if _node.name == node.name:
+            return n
+        n += 1
+    return 0
+
+
+def _get_network_from_rumba_experiment(exp):
+    print(exp)
+    print(exp.flows)
+    _nw = dict()
+    _nw['layers'] = dict()
+
+    _nw['nodes'] = list(range(len(exp.nodes)))
+
+    _nw['registrations'] = dict()
+
+    for node in exp.nodes:
+        for dif in node.difs:
+            if dif.name not in _nw['layers']:
+                _nw['layers'][dif.name] = dict()
+                _nw['layers'][dif.name]['type'] = rumba_to_type(dif.get_ipcp_class())
+                _nw['layers'][dif.name]['nodes'] = _get_nodes_in_dif(exp, dif.name)
+                _nw['layers'][dif.name]['edges'] = list()
+                if _nw['layers'][dif.name]['type'] != 'unicast':  # shim
+                    nodes = _nw['layers'][dif.name]['nodes']
+                    _nw['layers'][dif.name]['edges'].append((nodes[0], nodes[1]))
+                _nw['registrations'][dif.name] = dict()
+
+    for layer in exp.flows:
+        for flow in layer:
+            if 'src' in flow and 'dst' in flow:
+                src = _get_node_index(exp, flow['src'].node)
+                dst = _get_node_index(exp, flow['dst'].node)
+                layer = flow['src'].dif.name
+                _nw['layers'][layer]['edges'].append((src, dst))
+                src_regs = flow['src'].registrations
+                dst_regs = flow['dst'].registrations
+                for dif in src_regs:
+                    if dif.name not in _nw['registrations'][layer]:
+                        _nw['registrations'][layer][dif.name] = set()
+                    _nw['registrations'][layer][dif.name].add(src)
+                for dif in dst_regs:
+                    if dif.name not in _nw['registrations'][layer]:
+                        _nw['registrations'][layer][dif.name] = set()
+                    _nw['registrations'][layer][dif.name].add(dst)
+
+    return _nw
+
+
+def _get_rank(network, layer):
+    rank = 0
+
+    if layer not in network['registrations']:
+        return rank
+
+    for _layer in network['registrations'][layer]:
+        n_1_rank = _get_rank(network, _layer)
+        if n_1_rank >= rank:
+            rank = n_1_rank + 1
+
+    return rank
+
+
+def _get_ipcp_id(network, layer, node):
+    for ipcp in network['_ipcps']:
+        if ipcp['node'] == node and ipcp['layer'] == layer:
+            return ipcp['id']
+
+
+def _create_ipcp_network(network):
+    network['_ipcps'] = list()
+    network['_adjs'] = list()
+    network['_deps'] = list()
+    network['_systems'] = list()
+
+    for node in network['nodes']:
+        network['_systems'].append({'node': node})
+
+    ipcp_id = 0
+    color = 0
+
+    for layer in network['layers']:
+        for node in network['layers'][layer]['nodes']:
+            ipcp = {
+                'id': ipcp_id,
+                'node': node,
+                'layer': layer,
+                'color': color,
+                'type': network['layers'][layer]['type'],
+                'rank': _get_rank(network, layer)
+            }
+            network['_ipcps'].append(ipcp)
+            ipcp_id += 1
+        for edge in network['layers'][layer]['edges']:
+            src = _get_ipcp_id(network, layer, edge[0])
+            dst = _get_ipcp_id(network, layer, edge[1])
+            adj = {
+                'src': src,
+                'dst': dst,
+                'color': color
+            }
+            network['_adjs'].append(adj)
+        color += 1
+
+    for upper_layer in network['registrations']:
+        for lower_layer in network['registrations'][upper_layer]:
+            for node in network['registrations'][upper_layer][lower_layer]:
+                src = _get_ipcp_id(network, upper_layer, node)
+                dst = _get_ipcp_id(network, lower_layer, node)
+                network['_deps'].append((src, dst))
+
+
+def _create_system_graph(network):
+    edges = []
+
+    for layer in network['layers']:
+        for edge in network['layers'][layer]['edges']:
+            if edge not in edges:
+                edges.append(edge)
+
+    return edges
+
+
+def _create_system_coords(network):
+    """
+    Create coordinates for the systems in the network
+    :param network:
+    :return:
+    """
+    edges = _create_system_graph(network)
+
+    g = ig.Graph(edges, directed=False)
+    layout = g.layout('drl', dim=2)
+
+    for system in network['_systems']:
+        n = system['node']
+        system['coords'] = (layout[n][0], layout[n][1])
+
+    _min = None
+
+    for edge in edges:
+        x1, y1 = network['_systems'][edge[0]]['coords']
+        x2, y2 = network['_systems'][edge[1]]['coords']
+        d = sqrt((x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1))
+        _min = d if _min is None else min(d, _min)
+
+    return _min
+
+
+def _get_ipcps_for_system(network, system):
+    node = system['node']
+    return [ipcp for ipcp in network['_ipcps'] if ipcp['node'] == node]
+
+
+def _get_ranks_for_ipcps(ipcps):
+    return set([ipcp['rank'] for ipcp in ipcps])
+
+
+def _get_ipcps_by_rank(ipcps, rank):
+    return [ipcp for ipcp in ipcps if ipcp['rank'] == rank]
+
+
+def _create_ipcp_coords(network, radius):
+    """
+    Create 3d coordinates for IPCPs based on the system layout
+    """
+    height = 1
+
+    for system in network['_systems']:
+        ipcps = _get_ipcps_for_system(network, system)
+        ranks = _get_ranks_for_ipcps(ipcps)
+        for rank in ranks:
+            ri = _get_ipcps_by_rank(ipcps, rank)
+            n = len(ri)
+            m = 1
+            for ipcp in ri:
+                if n == 1:
+                    x, y = system['coords']
+                    ipcp['coords'] = (x, y, rank * height)
+                    continue
+                x = system['coords'][0]
+                y = system['coords'][1]
+                new_x = x + radius * sin((m/n) * pi)
+                new_y = y + radius * cos((m/n) * pi)
+                ipcp['coords'] = (new_x, new_y, rank * height)
+                m += 1
+
+
+def _create_ipcp_graph_data(network):
+    _create_ipcp_network(network)
+    _min = _create_system_coords(network)
+
+    print("_min = %s" % _min)
+    _create_ipcp_coords(network, _min / 5)
+
+    print(network)
+
+
+def _get_ipcp_attributes(network):
+    coords = list()
+    colors = list()
+    labels = list()
+    markers = list()
+
+    for ipcp in network['_ipcps']:
+        coords.append(ipcp['coords'])
+        colors.append(ipcp['color'])
+        labels.append(ipcp['layer'] + ' ' + str(ipcp['node']))
+        markers.append(type_to_marker[ipcp['type']])
+
+    return coords, colors, labels, markers
+
+
+def _get_edge_attributes(network):
+    x_coords = list()
+    y_coords = list()
+    z_coords = list()
+    colors = list()
+
+    for adj in network['_adjs']:
+        src = network['_ipcps'][adj['src']]['coords']
+        dst = network['_ipcps'][adj['dst']]['coords']
+        x_coords.extend([src[0], dst[0], None])
+        y_coords.extend([src[1], dst[1], None])
+        z_coords.extend([src[2], dst[2], None])
+        colors.append(adj['color'])
+
+    return x_coords, y_coords, z_coords, colors
+
+
+def _get_deps_attributes(network):
+    x_coords = list()
+    y_coords = list()
+    z_coords = list()
+    colors = list()
+
+    for dep in network['_deps']:
+        src = network['_ipcps'][dep[0]]['coords']
+        dst = network['_ipcps'][dep[1]]['coords']
+        x_coords.extend([src[0], dst[0], None])
+        y_coords.extend([src[1], dst[1], None])
+        z_coords.extend([src[2], dst[2], None])
+        colors.append(0)
+
+    return x_coords, y_coords, z_coords, colors
+
+
+def _extract(coords):
+    x = []
+    y = []
+    z = []
+
+    for coord in coords:
+        x.append(coord[0])
+        y.append(coord[1])
+        z.append(coord[2])
+
+    return x, y, z
+
+
+def draw_network(network, name='Ouroboros network'):
+    _create_ipcp_graph_data(network)
+    coords, colors, labels, markers = _get_ipcp_attributes(network)
+
+    x_coords, y_coords, z_coords = _extract(coords)
+
+    nodes = go.Scatter3d(
+        x=x_coords,
+        y=y_coords,
+        z=z_coords,
+        mode='markers+text',
+        name='actors',
+        marker=dict(symbol=markers,
+                    size=6,
+                    color=colors,
+                    colorscale='Viridis',
+                    line=dict(color='rgb(50,50,50)', width=0.5)
+                    ),
+        text=labels,
+        hoverinfo='none'
+    )
+
+    x_coords, y_coords, z_coords, colors = _get_edge_attributes(network)
+
+    layers = go.Scatter3d(
+        x=x_coords,
+        y=y_coords,
+        z=z_coords,
+        mode='lines',
+        line=dict(color=colors,
+                  colorscale='Viridis',
+                  dash='solid',
+                  width=2),
+        hoverinfo='none'
+    )
+
+    x_coords, y_coords, z_coords, colors = _get_deps_attributes(network)
+
+    deps = go.Scatter3d(
+        x=x_coords,
+        y=y_coords,
+        z=z_coords,
+        mode='lines',
+        line=dict(color=colors,
+                  colorscale='Viridis',
+                  dash='dashdot',
+                  width=1),
+        hoverinfo='none'
+    )
+
+    axis = dict(showbackground=False,
+                showline=False,
+                zeroline=False,
+                showgrid=False,
+                showticklabels=False,
+                title='')
+
+    layout = go.Layout(
+        title=name,
+        autosize=True,
+        width=3600,
+        height=1800,
+        showlegend=False,
+        scene=dict(
+            xaxis=dict(axis),
+            yaxis=dict(axis),
+            zaxis=dict(axis),
+        ),
+        margin=dict(
+            l=0,
+            r=0,
+            b=0,
+            t=100
+        ),
+        hovermode='closest',
+        annotations=[
+            dict(
+                showarrow=False,
+                text="Data source: Ouroboros",
+                xref='paper',
+                yref='paper',
+                xanchor='left',
+                yanchor='bottom',
+                x=0,
+                y=0.1,
+                font=dict(
+                    size=14
+                )
+            )
+        ],
+    )
+
+    data = [nodes, layers, deps]
+    fig = go.Figure(data=data, layout=layout)
+    fig.update_scenes(aspectmode='data')
+
+    fig.show()
+
+
+def draw_experiment(exp, name='experiment'):
+    _nw = _get_network_from_rumba_experiment(exp)
+    draw_network(_nw, name)
+
+
+if __name__ == '__main__':
+    nw = get_default_test_network()
+    draw_network(nw)