diff options
| author | Dimitri Staessens <dimitri@ouroboros.rocks> | 2026-03-01 11:02:25 +0100 |
|---|---|---|
| committer | Dimitri Staessens <dimitri@ouroboros.rocks> | 2026-03-07 11:29:02 +0100 |
| commit | 6cbcfb039e608419bd6ced673723918aca6fb278 (patch) | |
| tree | 172061d95f50cda09c3872d32c5f77e459044cd8 | |
| parent | 4e35c6b445d0cfbad9cf15a48f2d341e29dbd806 (diff) | |
| download | rumba-6cbcfb039e608419bd6ced673723918aca6fb278.tar.gz rumba-6cbcfb039e608419bd6ced673723918aca6fb278.zip | |
rumba: Remove irati/rlite, python2 and qemu support
Remove IRATI and rlite prototype plugins, keeping only Ouroboros.
Delete .gitlab-ci.yml (only contained an irati test job and a
Sphinx pages job). Clean up all irati/rlite imports and references
from examples, documentation, and tools.
Qemu was tied heavily with rlite and irati. As it's less useful for
ouroboros it's removed rather than reworked.
Updated README.md and AUTHORS
Signed-off-by: Dimitri Staessens <dimitri@ouroboros.rocks>
76 files changed, 3029 insertions, 4235 deletions
@@ -1,98 +1,47 @@ -# Byte-compiled / optimized / DLL files +# Byte-compiled / optimized __pycache__/ *.py[cod] *$py.class -# C extensions -*.so - # Distribution / packaging -.Python -env/ build/ -develop-eggs/ dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ *.egg-info/ -.installed.cfg *.egg +.eggs/ +sdist/ + +# Virtual environments +.venv/ +venv/ +env/ +ENV/ -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec +# IDE / editor +.idea/ +.vscode/ +*.swp +*.swo +*~ -# Installer logs -pip-log.txt -pip-delete-this-directory.txt +# Sphinx documentation +doc/_build/ # Unit test / coverage reports htmlcov/ .tox/ .coverage .coverage.* -.cache -nosetests.xml +.cache/ +.pytest_cache/ coverage.xml -*,cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# IPython Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# dotenv -.env - -# virtualenv -venv/ -ENV/ - -# Spyder project settings -.spyderproject - -# Rope project settings -.ropeproject +nosetests.xml -# emacs temporary files -*~ +# OS files +.DS_Store +Thumbs.db -*rspec +# Rumba runtime +/tmp/rumba/ *.pem - -# PyCharm metadata folder -.idea +rumba.log diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index dad6f1d..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,20 +0,0 @@ -before_script: - - apt-get update -qy - - apt-get install -y python-dev python-pip bridge-utils qemu - - pip install setuptools --upgrade - - python setup.py install - -irati-test: - script: - - python examples/vpn.py - -pages: - stage: deploy - script: - - pip install sphinx sphinx_rtd_theme - - sphinx-build doc/ public/ - artifacts: - paths: - - public - only: - - master @@ -0,0 +1,6 @@ +Sander Vrijders <sander@ouroboros.rocks> +Dimitri Staessens <dimitri@ouroboros.rocks> +Thijs Paelman <thijs@ouroboros.rocks> +Vincenzo Maffione <v.maffione@nextworks.it> +Marco Capitani <m.capitani@nextworks.it> +Nick Aerts <nick.aerts@ugent.be> diff --git a/AUTHORS.txt b/AUTHORS.txt deleted file mode 100644 index 0e929fb..0000000 --- a/AUTHORS.txt +++ /dev/null @@ -1,5 +0,0 @@ -Sander Vrijders <sander.vrijders@intec.ugent.be> -Dimitri Staessens <dimitri.staessens@ugent.be> -Vincenzo Maffione <v.maffione@nextworks.it> -Marco Capitani <m.capitani@nextworks.it> -Nick Aerts <nick.aerts@ugent.be>
\ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 21ede48..b44eea6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,3 @@ -include AUTHORS.txt -include LICENSE.txt -include README
\ No newline at end of file +include AUTHORS +include LICENSE +include README.md @@ -1,5 +1,208 @@ Rumba is a Python framework that allows users to write Python scripts to define recursive internet networks and run scripted experiments. -The official documentation can be found +The original documentation can be found [here](https://arcfire.gitlab.io/rumba/). + +This version of rumba is maintained by the Ouroboros (O7s) project. + +The documentation can be found +[here](https://ouroboros.rocks/wiki/Rumba). + +## Installation + +To install Rumba from a local clone: + +```bash +git clone https://codeberg.org/o7s/rumba.git +cd rumba +python3 -m venv .venv +source .venv/bin/activate +pip install . +``` + +Optional extras: + +```bash +pip install rumba[NumpyAcceleration] # faster random traffic generation +pip install rumba[graphs] # PDF graph export via pydot +pip install rumba[visualizer] # interactive visualization via igraph + plotly +``` + +On Debian/Ubuntu you may also need: + +```bash +sudo apt-get install build-essential libssl-dev libffi-dev python3-dev +``` + +The required dependencies (`paramiko` and `docker`) are installed +automatically. + +## Defining a Topology + +Rumba models recursive networks using **layers** and **nodes**. Layers +correspond to Ouroboros IPCP types. Nodes declare which layers they +belong to and how those layers are stacked via **registrations**. + +The example below defines four nodes connected in a ring, with a single +unicast layer (`n1`) running over four point-to-point Ethernet links: + +``` + a ---e01--- b + | | + e04 e02 + | | + d ---e03--- c +``` + +```python +from rumba.model import UnicastLayer, EthDixLayer, Node + +# A unicast layer that will span all four nodes +n1 = UnicastLayer("n1") + +# Four point-to-point Ethernet links (max 2 members each) +e01 = EthDixLayer("e01") # a <-> b +e02 = EthDixLayer("e02") # b <-> c +e03 = EthDixLayer("e03") # c <-> d +e04 = EthDixLayer("e04") # d <-> a + +# Each node lists its layer memberships and how layers are stacked. +# registrations = {upper: [lower, ...]} means "upper registers in lower". +a = Node("a", + layers=[n1, e01, e04], + registrations={n1: [e01, e04]}) + +b = Node("b", + layers=[n1, e01, e02], + registrations={n1: [e01, e02]}) + +c = Node("c", + layers=[n1, e02, e03], + registrations={n1: [e02, e03]}) + +d = Node("d", + layers=[n1, e03, e04], + registrations={n1: [e03, e04]}) +``` + +Two nodes that share an `EthDixLayer` are connected by that Ethernet +link. The `registrations` dictionary tells Rumba how to stack the +layers: `{n1: [e01, e04]}` means the unicast layer `n1` registers in +both `e01` and `e04` on that node, giving it connectivity over those +links. + +Policies can be set on unicast layers: + +```python +n1.add_policy("routing", "lfa") +``` + +Link quality (delay, loss, rate) can be configured on Ethernet layers: + +```python +from rumba.model import EthDixLayer, Delay + +e01 = EthDixLayer("e01") +e01.set_delay(delay=10, jitter=2) # 10 ms delay, 2 ms jitter +e01.set_loss(loss=0.1) # 0.1% packet loss +e01.set_rate(rate=100) # 100 Mbps +``` + +## Quick Start + +A complete runnable script that deploys the ring topology above on a +local testbed: + +```python +from rumba.model import * +from rumba.utils import ExperimentManager + +import rumba.testbeds.local as local +import rumba.prototypes.ouroboros as our + +# --- topology (from the example above) --- +n1 = UnicastLayer("n1") +e01 = EthDixLayer("e01") +e02 = EthDixLayer("e02") +e03 = EthDixLayer("e03") +e04 = EthDixLayer("e04") + +a = Node("a", layers=[n1, e01, e04], registrations={n1: [e01, e04]}) +b = Node("b", layers=[n1, e01, e02], registrations={n1: [e01, e02]}) +c = Node("c", layers=[n1, e02, e03], registrations={n1: [e02, e03]}) +d = Node("d", layers=[n1, e03, e04], registrations={n1: [e03, e04]}) + +# --- testbed and experiment --- +tb = local.Testbed() +exp = our.Experiment(tb, nodes=[a, b, c, d]) + +# --- run --- +with ExperimentManager(exp): + exp.swap_in() # provision the testbed + exp.bootstrap_prototype() # create IPCPs and enrollments +``` + +`swap_in()` provisions the nodes on the chosen testbed. +`bootstrap_prototype()` creates the Ouroboros IPCPs, bootstraps them, +and performs enrollments automatically. `ExperimentManager` handles +cleanup when the block exits. + +## Key Concepts + +- **Layer** — a network layer aligned with an Ouroboros IPCP type. + Available types: `UnicastLayer`, `BroadcastLayer`, `EthDixLayer`, + `EthLlcLayer`, `LocalLayer`, `Udp4Layer`, `Udp6Layer`. +- **Node** — a system declaring its layer memberships and + registrations (layer stacking). +- **Testbed** — the infrastructure backend that provisions nodes + (see table below). +- **Experiment** — computes IPCP placement, enrollment order, and + manages the prototype lifecycle. +- **StoryBoard** — schedules client/server traffic with support for + Poisson arrival rates. +- **CommandResult** — structured result from `node.execute_command()` + carrying exit code, stdout, and stderr. +- **Process** — a managed background process with `is_alive()`, + `wait()`, and `kill()`. +- **ExperimentManager** — context manager that handles experiment + setup and teardown. + +## Supported Testbeds + +| Testbed | Module | Description | +|----------|---------------------------|------------------------------------------------| +| Local | `rumba.testbeds.local` | Single machine, processes run locally | +| LocalNet | `rumba.testbeds.localnet` | Linux bridges + veth pairs, supports netem | +| Docker | `rumba.testbeds.dockertb` | Docker containers with bridge networking | +| Emulab | `rumba.testbeds.emulab` | Emulab / iMinds testbed (SSH-based) | +| jFed | `rumba.testbeds.jfed` | Virtual Wall, ExoGENI, CloudLab via jFed | + +## CLI Tools + +The `rumba-access` command opens an SSH session to an experiment node: + +```bash +rumba-access <node-name> +``` + +## More Examples + +See the [examples/](examples/) directory for additional topologies and +usage patterns, including multi-layer stacking, Docker deployments, +storyboard scripting, and large-scale experiments. + +## Citation + +If you use Rumba in your research, please cite: + +> S. Vrijders, D. Staessens, M. Capitani, and V. Maffione, +> "Rumba: A Python framework for automating large-scale recursive +> internet experiments on GENI and FIRE+," *IEEE*, 2018. + +## License + +Rumba is licensed under the +[GNU Lesser General Public License v2.1](LICENSE). + +See [AUTHORS](AUTHORS) for the list of contributors. diff --git a/doc/irati.rst b/doc/irati.rst deleted file mode 100644 index 0ad496a..0000000 --- a/doc/irati.rst +++ /dev/null @@ -1,11 +0,0 @@ -IRATI -============= - -`IRATI <https://github.com/IRATI/stack>`_ is an open source -implementation of the RINA architecture targeted at the OS/Linux -system, initially developed by the FP7-IRATI project. - -.. automodule:: rumba.prototypes.irati - :member-order: bysource - :show-inheritance: - :inherited-members: diff --git a/doc/model.rst b/doc/model.rst index 1ced817..454de7e 100644 --- a/doc/model.rst +++ b/doc/model.rst @@ -6,13 +6,21 @@ blocks listed below. .. autoclass:: rumba.model.Node -.. autoclass:: rumba.model.DIF +.. autoclass:: rumba.model.Layer -.. autoclass:: rumba.model.NormalDIF +.. autoclass:: rumba.model.UnicastLayer -.. autoclass:: rumba.model.ShimUDPDIF +.. autoclass:: rumba.model.BroadcastLayer -.. autoclass:: rumba.model.ShimEthDIF +.. autoclass:: rumba.model.EthDixLayer + +.. autoclass:: rumba.model.EthLlcLayer + +.. autoclass:: rumba.model.Udp4Layer + +.. autoclass:: rumba.model.Udp6Layer + +.. autoclass:: rumba.model.LocalLayer .. autoclass:: rumba.model.Distribution diff --git a/doc/prototype.rst b/doc/prototype.rst index a827dcb..0a17719 100644 --- a/doc/prototype.rst +++ b/doc/prototype.rst @@ -10,6 +10,4 @@ provided by the base class. Specific implementations of the Testbed class: .. toctree:: - irati - rlite ouroboros diff --git a/doc/qemu.rst b/doc/qemu.rst deleted file mode 100644 index 375ab55..0000000 --- a/doc/qemu.rst +++ /dev/null @@ -1,15 +0,0 @@ -QEMU -============= - -`QEMU <http://wiki.qemu-project.org/Main_Page>`_ is a generic and open -source machine emulator and virtualizer. - -In order to use the qemu testbed, the user should install the qemu and -bridge-utils packages on which the testbed depends: :: - - $ sudo apt-get install bridge-utils qemu - -.. automodule:: rumba.testbeds.qemu - :member-order: bysource - :show-inheritance: - :inherited-members: diff --git a/doc/rlite.rst b/doc/rlite.rst deleted file mode 100644 index 171c4b8..0000000 --- a/doc/rlite.rst +++ /dev/null @@ -1,11 +0,0 @@ -rlite -============= - -`rlite <https://gitlab.com/arcfire/rlite>`_ is a lightweight Free -and Open Source implementation of the Recursive InterNetwork -Architecture (RINA) for GNU/Linux operating systems. - -.. automodule:: rumba.prototypes.rlite - :member-order: bysource - :show-inheritance: - :inherited-members: diff --git a/doc/testbed.rst b/doc/testbed.rst index 0158904..45ac2c8 100644 --- a/doc/testbed.rst +++ b/doc/testbed.rst @@ -14,4 +14,3 @@ Specific implementations of the Testbed class: emulab jfed local - qemu diff --git a/doc/workflow.rst b/doc/workflow.rst index d0e56a7..1c35b91 100644 --- a/doc/workflow.rst +++ b/doc/workflow.rst @@ -2,14 +2,14 @@ Workflow ************************ 1. Define the network graph, creating instances of model.Node and - model.DIF classes. Experiments can import rumba.model to obtain all + model.Layer classes. Experiments can import rumba.model to obtain all Rumba classes needed to run an experiment. -2. Create an instance of a specific model.Testbed class (QEMU, Docker, +2. Create an instance of a specific model.Testbed class (Docker, Local, Emulab or jFed). 3. Create an instance of the selected prototype.Experiment class - (Ouroboros, rlite, IRATI), passing the testbed instance and a list + (Ouroboros), passing the testbed instance and a list of Node instances. * At the end of the base Experiment constructor, the diff --git a/examples/congestion.py b/examples/congestion.py new file mode 100755 index 0000000..844f136 --- /dev/null +++ b/examples/congestion.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python + +# An example script using the rumba package + +from rumba.model import Node, UnicastLayer, EthDixLayer +from rumba.storyboard import * +from rumba.utils import ExperimentManager, PROMPT_SWAPOUT +from rumba.topologies import build_star + +# import testbed plugins +import rumba.testbeds.jfed as jfed + +# import Ouroboros prototype plugin +import rumba.prototypes.ouroboros as our + +import rumba.log as log + + +__all__ = ["exp", "sb", "nodes", "main", "run"] + +log.set_logging_level('DEBUG') + +n1 = UnicastLayer("n1") + +leaves, routerNode = build_star( + ["client1", "client2", "server"], n1, hub_name="router") +clientNode1, clientNode2, serverNode = leaves + +nodes = ["client1", "client2", "router", "server"] + +tb = jfed.Testbed(exp_name='cc2', + cert_file='/path/to/cert.pem', + authority='wall1.ilabt.iminds.be', + image='UBUNTU18-64-STD', + username='username', + exp_hours='1', + proj_name='ouroborosrocks') + +exp = our.Experiment(tb, + nodes=[clientNode1, clientNode2, routerNode, serverNode], + # git_repo='https://codeberg.org/o7s/ouroboros', + git_branch='be', + build_options='-DCMAKE_BUILD_TYPE=Debug ' + '-DSHM_BUFFER_SIZE=131072 ' + '-DIPCP_ETH_PAD_RUNT_FRAMES=false ' + '-DDISABLE_CORE_LOCK=false ' + '-DPCP_SCHED_THR_MUL=1', + add_packages=['ethtool'], + influxdb={ + 'ip': '127.0.0.1', + 'port': 8086, + 'org': "Ouroboros", + 'token': "your-influxdb-token" + }) + +sb = StoryBoard(experiment=exp, duration=1500, servers=[]) + + +def run(): + sb.run_command("server", + 'irm bind prog ocbr name ocbr_server;' + 'irm name register ocbr_server layer n1;' + 'ocbr --listen > /dev/null 2>&1 &') + sb.run_command("client1", + 'irm bind prog ocbr name ocbr_client1;' + 'irm name register ocbr_client1 layer n1;' + 'ocbr --listen > /dev/null 2>&1 &') + sb.run_command("server", + 'irm bind prog oping name oping_server;' + 'irm name register oping_server layer n1;' + 'oping --listen > /dev/null 2>&1 &') + sb.run_command("client1", "for sz in `seq 0 10 200 1000`;" + "do oping -n oping_server -i 0ms -s $sz -d 120;" + "done") + sb.run_command("client1", "ocbr -n ocbr_server -r 70M -d 600 --spin > /dev/null 2>&1 &") + sb.run_command("server", "ocbr -n ocbr_client1 -r 70M -d 600 --spin > /dev/null 2>&1 &") + time.sleep(30) + sb.run_command("client2", "ocbr -n ocbr_server -r 70M -d 30 --spin > /dev/null 2>&1 &") + time.sleep(45) + sb.run_command("client2", "ocbr -n ocbr_server -r 20M -d 30 --spin > /dev/null 2>&1 &") + time.sleep(45) + sb.run_command("client2", "ocbr -n ocbr_server -r 60M -d 30 -s 1400 --spin > /dev/null 2>&1 &") + time.sleep(45) + sb.run_command("client2", "ocbr -n ocbr_server -r 60M -d 30 -s 1200 --spin > /dev/null 2>&1 &") + time.sleep(45) + sb.run_command("client2", "ocbr -n ocbr_server -r 60M -d 30 -s 1000 --spin > /dev/null 2>&1 &") + time.sleep(45) + sb.run_command("client2", "ocbr -n ocbr_server -r 60M -d 30 -s 800 --spin > /dev/null 2>&1 &") + time.sleep(35) + sb.run_command("client2", "ocbr -n ocbr_server -r 60M -d 30 -s 500 --spin > /dev/null 2>&1 &") + time.sleep(35) + sb.run_command("client2", "ocbr -n ocbr_server -r 60M -d 30 -s 200 --spin > /dev/null 2>&1 &") + time.sleep(200) +# sb.ouroboros_stop_metrics_exporter(nodes) +# exp.terminate_prototype() + + +def main(): + with ExperimentManager(exp, swap_out_strategy=PROMPT_SWAPOUT): + exp.swap_in() + exp.install_prototype() + exp.set_phy_link_rate_between("router", "server", 100) + exp.start_metrics_exporter(nodes, interval=300) + exp.bootstrap_prototype() + run() + + +if __name__ == "__main__": + main() diff --git a/examples/converged-operator-network.py b/examples/converged-operator-network.py index 42df95d..eb604f9 100755 --- a/examples/converged-operator-network.py +++ b/examples/converged-operator-network.py @@ -8,176 +8,171 @@ from rumba.storyboard import * from rumba.utils import * # import testbed plugins -import rumba.testbeds.emulab as emulab import rumba.testbeds.jfed as jfed -import rumba.testbeds.local as local -import rumba.testbeds.qemu as qemu # import prototype plugins import rumba.prototypes.ouroboros as our -import rumba.prototypes.rlite as rl -import rumba.prototypes.irati as irati log.set_logging_level('DEBUG') -f1 = NormalDIF("fixed") -l1 = NormalDIF("LTE") -m1 = NormalDIF("metro1") -m2 = NormalDIF("metro2") -m3 = NormalDIF("metro3") -c1 = NormalDIF("core") -n1 = NormalDIF("overlay") +f1 = UnicastLayer("fixed") +l1 = UnicastLayer("LTE") +m1 = UnicastLayer("metro1") +m2 = UnicastLayer("metro2") +m3 = UnicastLayer("metro3") +c1 = UnicastLayer("core") +n1 = UnicastLayer("overlay") -f1e1 = ShimEthDIF("f1e1") -f1e2 = ShimEthDIF("f1e2") -f1e3 = ShimEthDIF("f1e3") +f1e1 = EthDixLayer("f1e1") +f1e2 = EthDixLayer("f1e2") +f1e3 = EthDixLayer("f1e3") -l1e1 = ShimEthDIF("l1e1") +l1e1 = EthDixLayer("l1e1") -m1f1 = ShimEthDIF("m1f1") +m1f1 = EthDixLayer("m1f1") -m1e1 = ShimEthDIF("m1e1") -m1e2 = ShimEthDIF("m1e2") -m1e3 = ShimEthDIF("m1e3") -m1e4 = ShimEthDIF("m1e4") -m1e5 = ShimEthDIF("m1e5") -m1e6 = ShimEthDIF("m1e6") +m1e1 = EthDixLayer("m1e1") +m1e2 = EthDixLayer("m1e2") +m1e3 = EthDixLayer("m1e3") +m1e4 = EthDixLayer("m1e4") +m1e5 = EthDixLayer("m1e5") +m1e6 = EthDixLayer("m1e6") -m2l1 = ShimEthDIF("m2l1") +m2l1 = EthDixLayer("m2l1") -m2e1 = ShimEthDIF("m2e1") -m2e2 = ShimEthDIF("m2e2") -m2e3 = ShimEthDIF("m2e3") -m2e4 = ShimEthDIF("m2e4") -m2e5 = ShimEthDIF("m2e5") -m2e6 = ShimEthDIF("m2e6") +m2e1 = EthDixLayer("m2e1") +m2e2 = EthDixLayer("m2e2") +m2e3 = EthDixLayer("m2e3") +m2e4 = EthDixLayer("m2e4") +m2e5 = EthDixLayer("m2e5") +m2e6 = EthDixLayer("m2e6") -m3e1 = ShimEthDIF("m3e1") -m3e2 = ShimEthDIF("m3e2") -m3e3 = ShimEthDIF("m3e3") -m3e4 = ShimEthDIF("m3e4") -m3e5 = ShimEthDIF("m3e5") -m3e6 = ShimEthDIF("m3e6") +m3e1 = EthDixLayer("m3e1") +m3e2 = EthDixLayer("m3e2") +m3e3 = EthDixLayer("m3e3") +m3e4 = EthDixLayer("m3e4") +m3e5 = EthDixLayer("m3e5") +m3e6 = EthDixLayer("m3e6") -m1c1 = ShimEthDIF("m1c1") -m1c2 = ShimEthDIF("m1c2") +m1c1 = EthDixLayer("m1c1") +m1c2 = EthDixLayer("m1c2") -m2c1 = ShimEthDIF("m2c1") -m2c2 = ShimEthDIF("m2c2") +m2c1 = EthDixLayer("m2c1") +m2c2 = EthDixLayer("m2c2") -m3c1 = ShimEthDIF("m3c1") -m3c2 = ShimEthDIF("m3c2") +m3c1 = EthDixLayer("m3c1") +m3c2 = EthDixLayer("m3c2") -c1e1 = ShimEthDIF("c1e1") -c1e2 = ShimEthDIF("c1e2") -c1e3 = ShimEthDIF("c1e3") -c1e4 = ShimEthDIF("c1e4") +c1e1 = EthDixLayer("c1e1") +c1e2 = EthDixLayer("c1e2") +c1e3 = EthDixLayer("c1e3") +c1e4 = EthDixLayer("c1e4") f1n1 = Node("f1n1", - difs = [n1, f1, f1e3], - dif_registrations = {f1 : [f1e3], n1: [f1]}) + layers = [n1, f1, f1e3], + registrations = {f1 : [f1e3], n1: [f1]}) f1n2 = Node("f1n2", - difs = [f1, f1e1, f1e2, f1e3, m1f1], - dif_registrations = {f1: [f1e1, f1e2, f1e3, m1f1]}) + layers = [f1, f1e1, f1e2, f1e3, m1f1], + registrations = {f1: [f1e1, f1e2, f1e3, m1f1]}) f1n3 = Node("f1n3", - difs = [n1, f1, f1e1], - dif_registrations = {f1: [f1e1], n1: [f1]}) + layers = [n1, f1, f1e1], + registrations = {f1: [f1e1], n1: [f1]}) f1n4 = Node("f1n4", - difs = [n1, f1, f1e2], - dif_registrations = {f1: [f1e2], n1: [f1]}) + layers = [n1, f1, f1e2], + registrations = {f1: [f1e2], n1: [f1]}) l1n1 = Node("l1n1", - difs = [n1, l1, m2l1, l1e1], - dif_registrations = {l1: [m2l1, l1e1], n1: [l1]}) + layers = [n1, l1, m2l1, l1e1], + registrations = {l1: [m2l1, l1e1], n1: [l1]}) l1n2 = Node("l1n2", - difs = [n1, l1, l1e1], - dif_registrations = {l1: [l1e1], n1: [l1]}) + layers = [n1, l1, l1e1], + registrations = {l1: [l1e1], n1: [l1]}) m1n1 = Node("m1n1", - difs = [m1, m1e1, m1e6], - dif_registrations = {m1: [m1e1, m1e6]}) + layers = [m1, m1e1, m1e6], + registrations = {m1: [m1e1, m1e6]}) m1n2 = Node("m1n2", - difs = [m1, m1e1, m1e2], - dif_registrations = {m1: [m1e1, m1e2]}) + layers = [m1, m1e1, m1e2], + registrations = {m1: [m1e1, m1e2]}) m1n3 = Node("m1n3", - difs = [m1, m1e2, m1e3, m1c1], - dif_registrations = {m1: [m1e2, m1e3, m1c1]}) + layers = [m1, m1e2, m1e3, m1c1], + registrations = {m1: [m1e2, m1e3, m1c1]}) m1n4 = Node("m1n4", - difs = [m1, m1e3, m1e4, m1c2], - dif_registrations = {m1: [m1e3, m1e4, m1c2]}) + layers = [m1, m1e3, m1e4, m1c2], + registrations = {m1: [m1e3, m1e4, m1c2]}) m1n5 = Node("m1n5", - difs = [m1, m1e4, m1e5], - dif_registrations = {m1: [m1e4, m1e5]}) + layers = [m1, m1e4, m1e5], + registrations = {m1: [m1e4, m1e5]}) m1n6 = Node("m1n6", - difs = [n1, m1, f1, m1e5, m1e6, m1f1], - dif_registrations = {m1: [m1e5, m1e6], + layers = [n1, m1, f1, m1e5, m1e6, m1f1], + registrations = {m1: [m1e5, m1e6], f1: [m1f1], n1: [f1, m1]}) m2n1 = Node("m2n1", - difs = [m2, m2e1, m2e6], - dif_registrations = {m2: [m2e1, m2e6]}) + layers = [m2, m2e1, m2e6], + registrations = {m2: [m2e1, m2e6]}) m2n2 = Node("m2n2", - difs = [m2, m2e1, m2e2], - dif_registrations = {m2: [m2e1, m2e2]}) + layers = [m2, m2e1, m2e2], + registrations = {m2: [m2e1, m2e2]}) m2n3 = Node("m2n3", - difs = [m2, m2e2, m2e3, m2c1], - dif_registrations = {m2: [m2e2, m2e3, m2c1]}) + layers = [m2, m2e2, m2e3, m2c1], + registrations = {m2: [m2e2, m2e3, m2c1]}) m2n4 = Node("m2n4", - difs = [m2, m2e3, m2e4, m2c2], - dif_registrations = {m2: [m2e3, m2e4, m2c2]}) + layers = [m2, m2e3, m2e4, m2c2], + registrations = {m2: [m2e3, m2e4, m2c2]}) m2n5 = Node("m2n5", - difs = [m2, m2e4, m2e5], - dif_registrations = {m2: [m2e4, m2e5]}) + layers = [m2, m2e4, m2e5], + registrations = {m2: [m2e4, m2e5]}) m2n6 = Node("m2n6", - difs = [n1, m2, l1, m2e5, m2e6, m2l1], - dif_registrations = {m2: [m2e5, m2e6], l1: [m2l1], n1: [l1, m2]}) + layers = [n1, m2, l1, m2e5, m2e6, m2l1], + registrations = {m2: [m2e5, m2e6], l1: [m2l1], n1: [l1, m2]}) m3n1 = Node("m3n1", - difs = [m3, m3e1, m3e6], - dif_registrations = {m3: [m3e1, m3e6]}) + layers = [m3, m3e1, m3e6], + registrations = {m3: [m3e1, m3e6]}) m3n2 = Node("m3n2", - difs = [m3, m3e1, m3e2], - dif_registrations = {m3: [m3e1, m3e2]}) + layers = [m3, m3e1, m3e2], + registrations = {m3: [m3e1, m3e2]}) m3n3 = Node("m3n3", - difs = [m3, m3e2, m3e3, m3c1], - dif_registrations = {m3: [m3e2, m3e3, m3c1]}) + layers = [m3, m3e2, m3e3, m3c1], + registrations = {m3: [m3e2, m3e3, m3c1]}) m3n4 = Node("m3n4", - difs = [m3, m3e3, m3e4, m3c2], - dif_registrations = {m3: [m3e3, m3e4, m3c2]}) + layers = [m3, m3e3, m3e4, m3c2], + registrations = {m3: [m3e3, m3e4, m3c2]}) m3n5 = Node("m3n5", - difs = [m3, m3e4, m3e5], - dif_registrations = {m3: [m3e4, m3e5]}) + layers = [m3, m3e4, m3e5], + registrations = {m3: [m3e4, m3e5]}) m3n6 = Node("m3n6", - difs = [m3, m3e5, m3e6], - dif_registrations = {m3: [m3e5, m3e6]}) + layers = [m3, m3e5, m3e6], + registrations = {m3: [m3e5, m3e6]}) c1n1 = Node("c1n1", - difs = [n1, c1, m1, m2, m1c1, m2c1, c1e1, c1e4], - dif_registrations = {c1: [c1e1, c1e4], m1: [c1, m1c1], + layers = [n1, c1, m1, m2, m1c1, m2c1, c1e1, c1e4], + registrations = {c1: [c1e1, c1e4], m1: [c1, m1c1], m2: [c1, m2c1], n1: [m1, m2]}) c1n2 = Node("c1n2", - difs = [n1, c1, m2, m3, m2c2, m3c1, c1e1, c1e2], - dif_registrations = {c1: [c1e1, c1e2], m2: [c1, m2c2], + layers = [n1, c1, m2, m3, m2c2, m3c1, c1e1, c1e2], + registrations = {c1: [c1e1, c1e2], m2: [c1, m2c2], m3: [c1, m3c1], n1: [m2, m3]}) c1n3 = Node("c1n3", - difs = [n1, c1, m3, m3c2, c1e2, c1e3], - dif_registrations = {c1: [c1e2, c1e3], m3: [c1, m3c2], n1: [m3]}) + layers = [n1, c1, m3, m3c2, c1e2, c1e3], + registrations = {c1: [c1e2, c1e3], m3: [c1, m3c2], n1: [m3]}) c1n4 = Node("c1n4", - difs = [n1, c1, m1, m1c2, c1e3, c1e4], - dif_registrations = {c1: [c1e3, c1e4], m1: [c1, m1c2], n1: [m1]}) + layers = [n1, c1, m1, m1c2, c1e3, c1e4], + registrations = {c1: [c1e3, c1e4], m1: [c1, m1c2], n1: [m1]}) tb = jfed.Testbed(exp_name = "arcfiret43", proj_name = "rumba", - cert_file = "/home/dstaesse/jfed/cert.pem", + cert_file = "/path/to/cert.pem", authority = "exogeni.net", - username = "dstaesse") + username = "username") -exp = rl.Experiment(tb, nodes = [f1n1, f1n2, f1n3, f1n4, +exp = our.Experiment(tb, nodes = [f1n1, f1n2, f1n3, f1n4, l1n1, l1n2, m1n1, m1n2, m1n3, m1n4, m1n5, m1n6, m2n1, m2n2, m2n3, m2n4, m2n5, m2n6, @@ -190,10 +185,10 @@ with ExperimentManager(exp, swap_out_strategy=PAUSE_SWAPOUT): exp.swap_in() exp.install_prototype() exp.bootstrap_prototype() - c1 = Client("rinaperf", options ="-i 10000 -s 1000 -c 0 -d overlay", + c1 = Client("operf", options ="-s 1000 -d 0", nodes=[f1n1, f1n3, f1n4, l1n2]) - s1 = Server("rinaperf", arrival_rate=2, mean_duration=5, - options = "-l -d overlay", nodes = [c1n1], clients = [c1]) + s1 = Server("operf", arrival_rate=2, mean_duration=5, + options = "-l", nodes = [c1n1], clients = [c1]) sb = StoryBoard(experiment=exp, duration=3600, servers = [s1]) sb.generate_script() sb.start() diff --git a/examples/datacenter.py b/examples/datacenter.py new file mode 100755 index 0000000..ab02a2a --- /dev/null +++ b/examples/datacenter.py @@ -0,0 +1,78 @@ +from rumba.model import Node, UnicastLayer, EthDixLayer +import rumba.log as log +# import testbed plugins +import rumba.testbeds.local as local +from rumba.storyboard import * +# import Ouroboros prototype plugin +import rumba.prototypes.ouroboros as our + + +__all__ = ["local_exp", "name_nodes", "start_net"] + + +log.set_logging_level("DEBUG") + + +def buildRack(nRack, numHosts=2): + rack = UnicastLayer(f"rack{nRack}") + cables, hosts = [], [] + + for n in range(numHosts): + cables.append(EthDixLayer(f"e{n}r{nRack}")) + hosts.append( + Node( + f"h{n}r{nRack}", + layers=[cables[-1], rack], + registrations={rack: [cables[-1]]}, + ) + ) + + ToR_switch = Node( + f"s{nRack}", layers=[*cables, rack], registrations={rack: cables} + ) + + return [*hosts, ToR_switch] # list of nodes, ToR_switch latest + + +def build(numRacks=2, numHostsPerRack=2): + nodes, router_cables = [], [] + + for i in range(numRacks): + router_cables.append(EthDixLayer(f"router_{i}")) + nodes += buildRack(i, numHosts=numHostsPerRack) + nodes[-1].add_layer(router_cables[-1]) + #nodes[-1].add_registration(router_cables[-1]) + + Datacenter_router = Node("router", layers=router_cables) + + return [*nodes, Datacenter_router] + + +nodes = build() + +name_nodes = list(map(lambda node: node.name, nodes)) + +local_tb = local.Testbed() + + +local_exp = our.Experiment(local_tb, nodes=nodes) + + +def start_net(): + local_exp.swap_in() + local_exp.install_prototype() + local_exp.bootstrap_prototype() + + +def print_network(): + local_exp.export_connectivity_graph("datacenter_physical.pdf") + + for layer in local_exp.layer_ordering: + if not isinstance(layer, EthDixLayer): + local_exp.export_layer_graph(f"datacenter_layer_{layer.name}.pdf", layer) + + +if __name__ == '__main__': + #start_net() + print_network() + diff --git a/examples/docker-ouroboros.py b/examples/docker-ouroboros.py index ce30fdb..16f6f1d 100755 --- a/examples/docker-ouroboros.py +++ b/examples/docker-ouroboros.py @@ -4,6 +4,7 @@ from rumba.model import * from rumba.utils import ExperimentManager +from rumba.topologies import build_ring # import testbed plugins import rumba.testbeds.dockertb as docker @@ -25,33 +26,13 @@ args = argparser.parse_args() log.set_logging_level('DEBUG') -n01 = NormalDIF("n01") +n01 = UnicastLayer("n01") if (args.nodes < 3): print("The ouroboros must be longer than 2 nodes") sys.exit(-1) -nodes = [] - -shim_prev = None -for i in range(0, args.nodes): - shim = ShimEthDIF("e" + str(i)) - - if shim_prev == None and shim != None: - node = Node("node" + str(i), difs = [n01, shim], - dif_registrations = {n01 : [shim]}) - elif shim_prev != None and shim != None: - node = Node("node" + str(i), difs = [n01, shim, shim_prev], - dif_registrations = {n01 : [shim, shim_prev]}) - else: - node = Node("node" + str(i), difs = [n01, shim_prev], - dif_registrations = {n01 : [shim_prev]}) - - shim_prev = shim - nodes.append(node) - -nodes[0].add_dif(shim_prev) -nodes[0].add_dif_registration(n01, shim_prev) +nodes = build_ring(args.nodes, n01) tb = docker.Testbed(exp_name = "ouroboros") diff --git a/examples/example-script.rsb b/examples/example-script.rsb index 1d49a66..495bf4c 100644 --- a/examples/example-script.rsb +++ b/examples/example-script.rsb @@ -35,11 +35,11 @@ echo2, 18 &ev4| $sb run_client_of $Server.server_b # if no object ($ handle) is provided, the storyboard # is assumed as the object -14 | $Node.node_a set_link_state $ShimEthDIF.e1 'up' +14 | $Node.node_a set_link_state $EthLayer.e1 'up' -16 | $ShimEthDIF.e1 set_delay 30 10 +16 | $EthLayer.e1 set_delay 30 10 -28 | $ShimEthDIF.e1 set_loss 2 +28 | $EthLayer.e1 set_loss 2 diff --git a/examples/example.py b/examples/example.py index 2887cd5..9febd6e 100755 --- a/examples/example.py +++ b/examples/example.py @@ -7,59 +7,42 @@ from rumba.utils import ExperimentManager from rumba.storyboard import * # import testbed plugins -import rumba.testbeds.emulab as emulab import rumba.testbeds.jfed as jfed -import rumba.testbeds.local as local -import rumba.testbeds.qemu as qemu # import prototype plugins import rumba.prototypes.ouroboros as our -import rumba.prototypes.rlite as rl -import rumba.prototypes.irati as irati import rumba.log as log log.set_logging_level('DEBUG') -n1 = NormalDIF("n1") +n1 = UnicastLayer("n1") n1.add_policy("rmt.pff", "lfa") n1.add_policy("security-manager", "passwd") -e1 = ShimEthDIF("e1") +e1 = EthDixLayer("e1") a = Node("a", - difs=[n1, e1], - dif_registrations={n1: [e1]}) + layers=[n1, e1], + registrations={n1: [e1]}) b = Node("b", - difs=[e1, n1], - dif_registrations={n1: [e1]}) + layers=[e1, n1], + registrations={n1: [e1]}) tb = jfed.Testbed(exp_name="example1", username="user1", cert_file="/home/user1/cert.pem") -exp = rl.Experiment(tb, nodes=[a, b]) +exp = our.Experiment(tb, nodes=[a, b]) print(exp) # General setup (can be reused in other scripts as-is) storyboard = StoryBoard(duration=30) -# Clients can be applications that just keep running, and will be -# stopped by a SIGINT... -client1 = Client("rinaperf", - options="-t perf -s 1000 -c 0") - -# ... or a custom shutdown method can be provided. -client2 = Client("rinaperf", - options="-t perf -s 1000 -D <duration>", - shutdown="") - -server = Server("rinaperf", options="-l", arrival_rate=0.5, - mean_duration=5, clients=[client1, client2]) # Experiment-specific configuration: diff --git a/examples/geant.py b/examples/geant.py index 7fb59ad..720dd9e 100755 --- a/examples/geant.py +++ b/examples/geant.py @@ -4,16 +4,10 @@ from rumba.model import * # import testbed plugins -import rumba.testbeds.emulab as emulab import rumba.testbeds.jfed as jfed -import rumba.testbeds.local as local -import rumba.testbeds.qemu as qemu -import rumba.testbeds.dockertb as docker # import prototype plugins import rumba.prototypes.ouroboros as our -import rumba.prototypes.rlite as rl -import rumba.prototypes.irati as irati import rumba.log as log @@ -26,55 +20,55 @@ import time log.set_logging_level('DEBUG') -n1 = NormalDIF("n1") +n1 = UnicastLayer("n1") n1.add_policy("routing", "lfa") n1.add_policy("pff", "alternate") -e1 = ShimEthDIF("e1") -e2 = ShimEthDIF("e2") -e3 = ShimEthDIF("e3") -e4 = ShimEthDIF("e4") -e5 = ShimEthDIF("e5") -e6 = ShimEthDIF("e6") -e7 = ShimEthDIF("e7") -e8 = ShimEthDIF("e8") -e9 = ShimEthDIF("e9") -e10 = ShimEthDIF("e10") -e11 = ShimEthDIF("e11") -e12 = ShimEthDIF("e12") -e13 = ShimEthDIF("e13") -e14 = ShimEthDIF("e14") -e15 = ShimEthDIF("e15") -e16 = ShimEthDIF("e16") -e17 = ShimEthDIF("e17") -e18 = ShimEthDIF("e18") -e19 = ShimEthDIF("e19") -e20 = ShimEthDIF("e20") -e21 = ShimEthDIF("e21") -e22 = ShimEthDIF("e22") -e23 = ShimEthDIF("e23") -e24 = ShimEthDIF("e24") -e25 = ShimEthDIF("e25") -e26 = ShimEthDIF("e26") -e27 = ShimEthDIF("e27") -e28 = ShimEthDIF("e28") -e29 = ShimEthDIF("e29") -e30 = ShimEthDIF("e30") -e31 = ShimEthDIF("e31") -e32 = ShimEthDIF("e32") -e33 = ShimEthDIF("e33") -e34 = ShimEthDIF("e34") -e35 = ShimEthDIF("e35") -e36 = ShimEthDIF("e36") -e37 = ShimEthDIF("e37") -e38 = ShimEthDIF("e38") -e39 = ShimEthDIF("e39") -e40 = ShimEthDIF("e40") -e41 = ShimEthDIF("e41") -e42 = ShimEthDIF("e42") -e43 = ShimEthDIF("e43") -e44 = ShimEthDIF("e44") +e1 = EthDixLayer("e1") +e2 = EthDixLayer("e2") +e3 = EthDixLayer("e3") +e4 = EthDixLayer("e4") +e5 = EthDixLayer("e5") +e6 = EthDixLayer("e6") +e7 = EthDixLayer("e7") +e8 = EthDixLayer("e8") +e9 = EthDixLayer("e9") +e10 = EthDixLayer("e10") +e11 = EthDixLayer("e11") +e12 = EthDixLayer("e12") +e13 = EthDixLayer("e13") +e14 = EthDixLayer("e14") +e15 = EthDixLayer("e15") +e16 = EthDixLayer("e16") +e17 = EthDixLayer("e17") +e18 = EthDixLayer("e18") +e19 = EthDixLayer("e19") +e20 = EthDixLayer("e20") +e21 = EthDixLayer("e21") +e22 = EthDixLayer("e22") +e23 = EthDixLayer("e23") +e24 = EthDixLayer("e24") +e25 = EthDixLayer("e25") +e26 = EthDixLayer("e26") +e27 = EthDixLayer("e27") +e28 = EthDixLayer("e28") +e29 = EthDixLayer("e29") +e30 = EthDixLayer("e30") +e31 = EthDixLayer("e31") +e32 = EthDixLayer("e32") +e33 = EthDixLayer("e33") +e34 = EthDixLayer("e34") +e35 = EthDixLayer("e35") +e36 = EthDixLayer("e36") +e37 = EthDixLayer("e37") +e38 = EthDixLayer("e38") +e39 = EthDixLayer("e39") +e40 = EthDixLayer("e40") +e41 = EthDixLayer("e41") +e42 = EthDixLayer("e42") +e43 = EthDixLayer("e43") +e44 = EthDixLayer("e44") layers = [e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13, e14, e15, e16, e17, e18, e19, e20, @@ -83,130 +77,129 @@ layers = [e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e41, e42, e43, e44] lisbon = Node("lisbon", - difs = [n1, e1, e2], - dif_registrations = {n1: [e1,e2]}) + layers = [n1, e1, e2], + registrations = {n1: [e1,e2]}) madrid = Node("madrid", - difs = [n1, e1, e6, e9, e11], - dif_registrations = {n1 : [e1, e6, e9, e11]}) + layers = [n1, e1, e6, e9, e11], + registrations = {n1 : [e1, e6, e9, e11]}) london = Node("london", - difs = [n1, e2, e3, e4, e5], - dif_registrations = {n1 : [e2, e3, e4, e5]}) + layers = [n1, e2, e3, e4, e5], + registrations = {n1 : [e2, e3, e4, e5]}) dublin = Node("dublin", - difs = [n1, e3], - dif_registrations = {n1 : [e3]}) + layers = [n1, e3], + registrations = {n1 : [e3]}) paris = Node("paris", - difs = [n1, e4, e6, e7, e8], - dif_registrations = {n1 : [e4, e6, e7, e8]}) + layers = [n1, e4, e6, e7, e8], + registrations = {n1 : [e4, e6, e7, e8]}) brussels = Node("brussels", - difs = [n1, e5, e12], - dif_registrations = {n1 : [e5, e12]}) + layers = [n1, e5, e12], + registrations = {n1 : [e5, e12]}) luxemburg = Node("luxemburg", - difs = [n1, e7, e16], - dif_registrations = {n1 : [e7, e16]}) + layers = [n1, e7, e16], + registrations = {n1 : [e7, e16]}) bern = Node("bern", - difs = [n1, e8, e9, e10, e15], - dif_registrations = {n1 : [e8, e9, e10, e15]}) + layers = [n1, e8, e9, e10, e15], + registrations = {n1 : [e8, e9, e10, e15]}) roma = Node("roma", - difs = [n1, e10, e11, e13, e34, e28], - dif_registrations = {n1 : [e10, e11, e13, e34, e28]}) + layers = [n1, e10, e11, e13, e34, e28], + registrations = {n1 : [e10, e11, e13, e34, e28]}) amsterdam = Node("amsterdam", - difs = [n1, e12, e14, e17, e18], - dif_registrations = {n1 : [e12, e14, e17, e18]}) + layers = [n1, e12, e14, e17, e18], + registrations = {n1 : [e12, e14, e17, e18]}) valleta = Node("valleta", - difs = [n1, e13, e14], - dif_registrations = {n1 : [e13, e14]}) + layers = [n1, e13, e14], + registrations = {n1 : [e13, e14]}) berlin = Node("berlin", - difs = [n1, e15, e16, e17, e30, e31], - dif_registrations = {n1 : [e15, e16, e17, e30, e31]}) + layers = [n1, e15, e16, e17, e30, e31], + registrations = {n1 : [e15, e16, e17, e30, e31]}) copenhagen = Node("copenhagen", - difs = [n1, e18, e20, e22, e24, e19], - dif_registrations = {n1 : [e18, e20, e22, e24, e19]}) + layers = [n1, e18, e20, e22, e24, e19], + registrations = {n1 : [e18, e20, e22, e24, e19]}) oslo = Node("oslo", - difs = [n1, e20, e21], - dif_registrations = {n1 : [e20, e21]}) + layers = [n1, e20, e21], + registrations = {n1 : [e20, e21]}) stockholm = Node("stockholm", - difs = [n1, e21, e22], - dif_registrations = {n1 : [e21, e22]}) + layers = [n1, e21, e22], + registrations = {n1 : [e21, e22]}) tallin = Node("tallin", - difs = [n1, e24, e25], - dif_registrations = {n1 : [e24, e25]}) + layers = [n1, e24, e25], + registrations = {n1 : [e24, e25]}) riga = Node("riga", - difs = [n1, e25, e26], - dif_registrations = {n1 : [e25, e26]}) + layers = [n1, e25, e26], + registrations = {n1 : [e25, e26]}) vilnius = Node("vilnius", - difs = [n1, e26, e27], - dif_registrations = {n1 : [e26, e27]}) + layers = [n1, e26, e27], + registrations = {n1 : [e26, e27]}) warsaw = Node("warsaw", - difs = [n1, e27, e29], - dif_registrations = {n1 : [e27, e29]}) + layers = [n1, e27, e29], + registrations = {n1 : [e27, e29]}) praha = Node("praha", - difs = [n1, e29, e30, e32], - dif_registrations = {n1 : [e29, e30, e32]}) + layers = [n1, e29, e30, e32], + registrations = {n1 : [e29, e30, e32]}) viena = Node("viena", - difs = [n1, e32, e33, e35, e39, e28], - dif_registrations = {n1 : [e32, e33, e35, e39, e28]}) + layers = [n1, e32, e33, e35, e39, e28], + registrations = {n1 : [e32, e33, e35, e39, e28]}) budapest = Node("budapest", - difs = [n1, e33, e37, e38, e43], - dif_registrations = {n1 : [e33, e37, e38, e43]}) + layers = [n1, e33, e37, e38, e43], + registrations = {n1 : [e33, e37, e38, e43]}) athens = Node("athens", - difs = [n1, e39, e40, e44], - dif_registrations = {n1 : [e39, e40, e44]}) + layers = [n1, e39, e40, e44], + registrations = {n1 : [e39, e40, e44]}) ljubljana = Node("ljubljana", - difs = [n1, e35, e36], - dif_registrations = {n1 : [e35, e36]}) + layers = [n1, e35, e36], + registrations = {n1 : [e35, e36]}) zagreb = Node("zagreb", - difs = [n1, e36, e37], - dif_registrations = {n1 : [e36, e37]}) + layers = [n1, e36, e37], + registrations = {n1 : [e36, e37]}) sofia = Node("sofia", - difs = [n1, e38, e40, e42, e23], - dif_registrations = {n1 : [e38, e40, e42, e23]}) + layers = [n1, e38, e40, e42, e23], + registrations = {n1 : [e38, e40, e42, e23]}) bucharest = Node("bucharest", - difs = [n1, e42, e43, e41], - dif_registrations = {n1 : [e42, e43, e41]}) + layers = [n1, e42, e43, e41], + registrations = {n1 : [e42, e43, e41]}) nicosia = Node("nicosia", - difs = [n1, e44, e34], - dif_registrations = {n1 : [e44, e34]}) + layers = [n1, e44, e34], + registrations = {n1 : [e44, e34]}) ankara = Node("ankara", - difs = [n1, e23, e41], - dif_registrations = {n1 : [e23, e41]}) + layers = [n1, e23, e41], + registrations = {n1 : [e23, e41]}) moscow = Node("moscow", - difs = [n1, e31, e19], - dif_registrations = {n1 : [e31, e19]}) + layers = [n1, e31, e19], + registrations = {n1 : [e31, e19]}) tb = jfed.Testbed(exp_name = 'geant8', - cert_file = '/home/sander/cert.pem', + cert_file = '/path/to/cert.pem', authority = 'wall1.ilabt.iminds.be', -# authority = 'exogeni.net:umassvmsite', - username = 'sander', + username = 'username', exp_hours = '200') nodes = [lisbon, madrid, london, paris, dublin, @@ -217,7 +210,7 @@ nodes = [lisbon, madrid, london, paris, dublin, bucharest, nicosia, ankara, moscow] exp = our.Experiment(tb, nodes=nodes, - git_repo='https://bitbucket.org/sandervrijders/ouroboros.git', + git_repo='https://codeberg.org/o7s/ouroboros', git_branch='rand3') @@ -241,7 +234,7 @@ with ExperimentManager(exp, swap_out_strategy=AUTO_SWAPOUT): arrival_rate=0.01, mean_duration=duration, s_id='oping_' + node.name, - difs=n1 + layers=n1 ) sb.add_server_on_node(s, node) @@ -294,9 +287,9 @@ with ExperimentManager(exp, swap_out_strategy=AUTO_SWAPOUT): prefix = layer.name + '_' + str(i) sb.schedule_export_lsdb_total(390, prefix + '_lsdb.csv', n1) - sb.schedule_export_dif_bandwidth(410, prefix + '_1.csv', n1) - sb.schedule_export_dif_bandwidth(510, prefix + '_2.csv', n1) - sb.schedule_export_dif_bandwidth(610, prefix + '_3.csv', n1) - sb.schedule_export_dif_bandwidth(710, prefix + '_4.csv', n1) + sb.schedule_export_layer_bandwidth(410, prefix + '_1.csv', n1) + sb.schedule_export_layer_bandwidth(510, prefix + '_2.csv', n1) + sb.schedule_export_layer_bandwidth(610, prefix + '_3.csv', n1) + sb.schedule_export_layer_bandwidth(710, prefix + '_4.csv', n1) sb.start() diff --git a/examples/isps.py b/examples/isps.py index 62a17ad..335c43b 100755 --- a/examples/isps.py +++ b/examples/isps.py @@ -15,23 +15,23 @@ import time log.set_logging_level('DEBUG') -n0 = NormalDIF("n0") +n0 = UnicastLayer("n0") -n1 = NormalDIF("n1") +n1 = UnicastLayer("n1") -n2 = NormalDIF("n2") +n2 = UnicastLayer("n2") n2.add_policy("routing", "lfa") n2.add_policy("pff", "alternate") -e0 = ShimEthDIF("e0") -e1 = ShimEthDIF("e1") -e2 = ShimEthDIF("e2") -e3 = ShimEthDIF("e3") -e4 = ShimEthDIF("e4") -e5 = ShimEthDIF("e5") -e6 = ShimEthDIF("e6") -e7 = ShimEthDIF("e7") -e8 = ShimEthDIF("e8") +e0 = EthDixLayer("e0") +e1 = EthDixLayer("e1") +e2 = EthDixLayer("e2") +e3 = EthDixLayer("e3") +e4 = EthDixLayer("e4") +e5 = EthDixLayer("e5") +e6 = EthDixLayer("e6") +e7 = EthDixLayer("e7") +e8 = EthDixLayer("e8") es = [e0, e1, e2, e3, e4, e5, e6, e7, e8] @@ -39,50 +39,50 @@ for e in es: e.set_delay(5) end0 = Node("end0", - difs = [e0, e8, n2], - dif_registrations = {n2: [e0, e8]}) + layers = [e0, e8, n2], + registrations = {n2: [e0, e8]}) end1 = Node("end1", - difs = [e4, e5, n2], - dif_registrations = {n2: [e4, e5]}) + layers = [e4, e5, n2], + registrations = {n2: [e4, e5]}) isp10 = Node("isp10", - difs = [e0, e1, n2, n1], - dif_registrations = {n2: [e0, n1], n1: [e1]}) + layers = [e0, e1, n2, n1], + registrations = {n2: [e0, n1], n1: [e1]}) isp11 = Node("isp11", - difs = [e1, e2, n1], - dif_registrations = {n1: [e1, e2]}) + layers = [e1, e2, n1], + registrations = {n1: [e1, e2]}) isp12 = Node("isp12", - difs = [e2, e3, n1], - dif_registrations = {n1: [e2, e3]}) + layers = [e2, e3, n1], + registrations = {n1: [e2, e3]}) isp13 = Node("isp13", - difs = [e3, e4, n2, n1], - dif_registrations = {n2: [e4, n1], n1: [e3]}) + layers = [e3, e4, n2, n1], + registrations = {n2: [e4, n1], n1: [e3]}) isp00 = Node("isp00", - difs = [e8, e7, n2, n0], - dif_registrations = {n2: [e8, n0], n0: [e7]}) + layers = [e8, e7, n2, n0], + registrations = {n2: [e8, n0], n0: [e7]}) isp01 = Node("isp01", - difs = [e7, e6, n0], - dif_registrations = {n0: [e7, e6]}) + layers = [e7, e6, n0], + registrations = {n0: [e7, e6]}) isp02 = Node("isp02", - difs = [e6, e5, n2, n0], - dif_registrations = {n2: [e5, n0], n0: [e6]}) + layers = [e6, e5, n2, n0], + registrations = {n2: [e5, n0], n0: [e6]}) tb = jfed.Testbed(exp_name = 'case3', - cert_file = '/home/sander/cert.pem', - username = 'sander', + cert_file = '/path/to/cert.pem', + username = 'username', exp_hours = '1') nodes = [end0, end1, isp10, isp11, isp12, isp13, isp00, isp01, isp02] exp = our.Experiment(tb, nodes=nodes, - git_repo='https://bitbucket.org/sandervrijders/ouroboros.git', + git_repo='https://codeberg.org/o7s/ouroboros', git_branch='rand3') duration = 120 @@ -97,7 +97,7 @@ s = Server( arrival_rate=0.01, mean_duration=duration + 4, s_id='oping', - difs=n2 + layers=n2 ) sb.add_server_on_node(s, end1) diff --git a/examples/jfed-rlite.py b/examples/jfed-rlite.py deleted file mode 100755 index 5cc087f..0000000 --- a/examples/jfed-rlite.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python - -from rumba.model import * -from rumba.utils import ExperimentManager - -import rumba.testbeds.jfed as jfed -import rumba.prototypes.rlite as rlite - -import rumba.log as log - -import argparse - - -description = "Script to run rlite on jfed" -epilog = "2017 H2020 ARCFIRE" - -argparser = argparse.ArgumentParser(description = description, - epilog = epilog) -argparser.add_argument('--user', type = str, default = 'vmaffio', - help = "jFed username") -argparser.add_argument('--cert', type = str, - help = "Absolute path to certificate (.pem) file" - " to be used with jFed", - default = '/home/vmaffione/Downloads/vmaffio-jfed.pem') -argparser.add_argument('--expname', type = str, default = 'pinocchio', - help = "Name of the experiment within the jFed testbed") - -args = argparser.parse_args() - -log.set_logging_level('DEBUG') - -n1 = NormalDIF("n1") - -e1 = ShimEthDIF("e1") - -a = Node("a", - difs = [n1, e1], - dif_registrations = {n1 : [e1]}) - -b = Node("b", - difs = [e1, n1], - dif_registrations = {n1 : [e1]}) - -tb = jfed.Testbed(exp_name = args.expname, - cert_file = args.cert, - username = args.user) - -exp = rlite.Experiment(tb, nodes = [a, b]) - -with ExperimentManager(exp): - exp.swap_in() - exp.install_prototype() - exp.bootstrap_prototype() diff --git a/examples/mouse.py b/examples/mouse.py index d80da28..38be6e0 100755 --- a/examples/mouse.py +++ b/examples/mouse.py @@ -3,107 +3,106 @@ # An example script using the rumba package from rumba.model import * -from rumba.utils import ExperimentManager +from rumba.utils import ExperimentManager, PROMPT_SWAPOUT # import testbed plugins -import rumba.testbeds.emulab as emulab -import rumba.testbeds.jfed as jfed import rumba.testbeds.local as local -import rumba.testbeds.qemu as qemu # import prototype plugins import rumba.prototypes.ouroboros as our -import rumba.prototypes.rlite as rl -import rumba.prototypes.irati as irati import rumba.log as log +from rumba.visualizer import draw_network, get_network_from_rumba_experiment + log.set_logging_level('DEBUG') -n01 = NormalDIF("n01") +n01 = UnicastLayer("n01") -e01 = ShimEthDIF("e01") -e02 = ShimEthDIF("e02") -e03 = ShimEthDIF("e03") -e04 = ShimEthDIF("e04") -e05 = ShimEthDIF("e05") -e06 = ShimEthDIF("e06") -e07 = ShimEthDIF("e07") -e08 = ShimEthDIF("e08") -e09 = ShimEthDIF("e09") -e10 = ShimEthDIF("e10") -e11 = ShimEthDIF("e11") -e12 = ShimEthDIF("e12") -e13 = ShimEthDIF("e13") -e14 = ShimEthDIF("e14") -e15 = ShimEthDIF("e15") -e16 = ShimEthDIF("e16") -e17 = ShimEthDIF("e17") +e01 = EthDixLayer("e01") +e02 = EthDixLayer("e02") +e03 = EthDixLayer("e03") +e04 = EthDixLayer("e04") +e05 = EthDixLayer("e05") +e06 = EthDixLayer("e06") +e07 = EthDixLayer("e07") +e08 = EthDixLayer("e08") +e09 = EthDixLayer("e09") +e10 = EthDixLayer("e10") +e11 = EthDixLayer("e11") +e12 = EthDixLayer("e12") +e13 = EthDixLayer("e13") +e14 = EthDixLayer("e14") +e15 = EthDixLayer("e15") +e16 = EthDixLayer("e16") +e17 = EthDixLayer("e17") a = Node("a", - difs = [n01, e01, e06, e13 ], - dif_registrations = {n01 : [e01, e06, e13]}) + layers = [n01, e01, e06, e13 ], + registrations = {n01 : [e01, e06, e13]}) b = Node("b", - difs = [n01, e01, e02, e04], - dif_registrations = {n01 : [e01, e02, e04]}) + layers = [n01, e01, e02, e04], + registrations = {n01 : [e01, e02, e04]}) c = Node("c", - difs = [n01, e02, e03], - dif_registrations = {n01 : [e02, e03]}) + layers = [n01, e02, e03], + registrations = {n01 : [e02, e03]}) d = Node("d", - difs = [n01, e03, e04, e05], - dif_registrations = {n01 : [e03, e04, e05]}) + layers = [n01, e03, e04, e05], + registrations = {n01 : [e03, e04, e05]}) e = Node("e", - difs = [n01, e05, e06, e07], - dif_registrations = {n01 : [e05, e06, e07]}) + layers = [n01, e05, e06, e07], + registrations = {n01 : [e05, e06, e07]}) f = Node("f", - difs = [n01, e07, e08], - dif_registrations = {n01 : [e07, e08]}) + layers = [n01, e07, e08], + registrations = {n01 : [e07, e08]}) g = Node("g", - difs = [n01, e08, e09, e14], - dif_registrations = {n01 : [e08, e09, e14]}) + layers = [n01, e08, e09, e14], + registrations = {n01 : [e08, e09, e14]}) h = Node("h", - difs = [n01, e09, e10, e15], - dif_registrations = {n01 : [e09, e10, e15]}) + layers = [n01, e09, e10, e15], + registrations = {n01 : [e09, e10, e15]}) i = Node("i", - difs = [n01, e10, e11, e16], - dif_registrations = {n01 : [e10, e11, e16]}) + layers = [n01, e10, e11, e16], + registrations = {n01 : [e10, e11, e16]}) j = Node("j", - difs = [n01, e11, e12], - dif_registrations = {n01 : [e11, e12]}) + layers = [n01, e11, e12], + registrations = {n01 : [e11, e12]}) k = Node("k", - difs = [n01, e12, e13], - dif_registrations = {n01 : [e12, e13]}) + layers = [n01, e12, e13], + registrations = {n01 : [e12, e13]}) l = Node("l", - difs = [n01, e14, e15], - dif_registrations = {n01 : [e14, e15]}) + layers = [n01, e14, e15], + registrations = {n01 : [e14, e15]}) m = Node("m", - difs = [n01, e16, e17], - dif_registrations = {n01 : [e16, e17]}) + layers = [n01, e16, e17], + registrations = {n01 : [e16, e17]}) n = Node("n", - difs = [n01, e17], - dif_registrations = {n01 : [e17]}) + layers = [n01, e17], + registrations = {n01 : [e17]}) -tb = qemu.Testbed(exp_name = "mouse2") +tb = local.Testbed(exp_name = "mouse2") -exp = rl.Experiment(tb, nodes = [a, b, c, d, e, f, g, h, i, j, k, l, m, n]) +exp = our.Experiment(tb, nodes = [a, b, c, d, e, f, g, h, i, j, k, l, m, n]) print(exp) -with ExperimentManager(exp): +with ExperimentManager(exp, swap_out_strategy=PROMPT_SWAPOUT): + nw = get_network_from_rumba_experiment(exp) + draw_network(nw) exp.swap_in() exp.bootstrap_prototype() diff --git a/examples/ouroboros-layer-example.py b/examples/ouroboros-layer-example.py new file mode 100644 index 0000000..310a204 --- /dev/null +++ b/examples/ouroboros-layer-example.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python + +""" +Example using the Ouroboros-aligned layer-centric API. + +Shows the naming conventions: Layer, UnicastLayer, EthDixLayer, etc. +""" + +from rumba.model import * +from rumba.utils import ExperimentManager + +# import testbed plugins +import rumba.testbeds.local as local + +# import prototype plugins +import rumba.prototypes.ouroboros as our + +import rumba.log as log + +log.set_logging_level('DEBUG') + +# ----------------------------------------------------------------------- +# Layer-centric syntax +# ----------------------------------------------------------------------- + +# Define layers +top = UnicastLayer("top") +mid = UnicastLayer("mid") +top.add_policy("rmt.pff", "lfa") + +# Ethernet layers model point-to-point links +e1 = EthDixLayer("e1") +e2 = EthDixLayer("e2") +e3 = EthDixLayer("e3") + +# Define nodes and their layer memberships + registrations +a = Node("a", + layers=[top, mid, e1, e2], + registrations={mid: [e1, e2], + top: [mid]}) + +b = Node("b", + layers=[mid, e1, e3], + registrations={mid: [e1, e3], + }) + +c = Node("c", + layers=[top, mid, e2, e3], + registrations={mid: [e2, e3], + top: [mid]}) + +# Create testbed and experiment +tb = local.Testbed(exp_name="layer-example") +exp = our.Experiment(tb, nodes=[a, b, c]) + +print(exp) + +# Use properties from the API +for layer in exp.layer_ordering: + print("Layer: %s (type=%s, is_eth=%s, is_shim=%s)" % ( + layer.name, layer.layer_type.value, layer.is_eth, layer.is_shim)) + +with ExperimentManager(exp): + exp.swap_in() + exp.install_prototype() + exp.bootstrap_prototype() diff --git a/examples/rumba_example.py b/examples/rumba_example.py new file mode 100755 index 0000000..2027ffa --- /dev/null +++ b/examples/rumba_example.py @@ -0,0 +1,23 @@ +from rumba.model import Node, UnicastLayer, EthDixLayer +from rumba.topologies import build_star + +# import testbed plugins +import rumba.testbeds.local as local + +# import Ouroboros prototype plugin +import rumba.prototypes.ouroboros as our + +__all__ = ["exp", "nodes"] + +n1 = UnicastLayer("n1") + +leaves, routerNode = build_star( + ["client1", "client2", "server"], n1, hub_name="router") +clientNode1, clientNode2, serverNode = leaves + +nodes = ["client1", "client2", "router", "server"] + +tb = local.Testbed() +exp = our.Experiment(tb, + nodes=[clientNode1, clientNode2, + routerNode, serverNode]) diff --git a/examples/scalingtime.py b/examples/scalingtime.py index 920553c..9490b4f 100755 --- a/examples/scalingtime.py +++ b/examples/scalingtime.py @@ -1,23 +1,19 @@ -#!/usr/bin/env python - from rumba.model import * from rumba.utils import * # import testbed plugins import rumba.testbeds.emulab as emulab import rumba.testbeds.jfed as jfed -import rumba.testbeds.qemu as qemu import rumba.testbeds.local as local import rumba.testbeds.dockertb as docker # import prototype plugins import rumba.prototypes.ouroboros as our -import rumba.prototypes.rlite as rl -import rumba.prototypes.irati as irati import argparse +import json -from rumba.visualizer import draw_experiment +from rumba.visualizer import draw_network, get_network_from_rumba_experiment description = "Script to create the demo from the Rumba paper" @@ -41,29 +37,16 @@ argparser.add_argument('--swapout', type=SwapOutStrategy.from_string, default=PROMPT_SWAPOUT, choices=list(SwapOutStrategy), help='What action to perform on swap-out') -argparser.add_argument('--prototype', type=str, required=True, - choices=['irati', 'ouroboros', 'rlite'], - help='The kind of prototype plugin to use to run' - ' the experiment.') +argparser.add_argument('--visualize', action="store_true", + help='Draw a representation of the network') + subparsers = argparser.add_subparsers(dest='testbed') emulab_p = subparsers.add_parser('emulab', help='Use emulab testbed') jfed_p = subparsers.add_parser('jfed', help='Use jfed testbed') -qemu_p = subparsers.add_parser('qemu', help='Use qemu testbed') local_p = subparsers.add_parser('local', help='Use local testbed') docker_p = subparsers.add_parser('docker', help='Use Docker testbed') -qemu_p.add_argument('--bzimage', metavar='BZIMAGE', type=str, - help='path to the bzImage file to use') -qemu_p.add_argument('--initramfs', metavar='INITRAMFS', type=str, - help='path to the initramfs file to use') -qemu_p.add_argument('--use_vhost', action='store_true', - default=False, help='Use vhost') -qemu_p.add_argument('--qemu_logs_dir', metavar='QEMU_LOGS', type=str, - default=None, help='path to the folder for qemu logs') -qemu_p.add_argument('--username', type=str, help="Username") -qemu_p.add_argument('--password', type=str, help="Password") - emulab_p.add_argument('--url', metavar='URL', type=str, default="wall2.ilabt.iminds.be", help='Url') @@ -102,91 +85,90 @@ args = argparser.parse_args() log.set_logging_level(args.verbosity) -pi = NormalDIF("pi") -pi.dt_strategy = 'minimal' +pi = UnicastLayer("pi") -c = NormalDIF("core") +c = UnicastLayer("core") nodes = [] core_nodes = [] for i in range(0, args.metro_networks): - m = NormalDIF("m-" + str(i)) + m = UnicastLayer("m-" + str(i)) e_prev = None m1 = None mn = None for j in range(0, args.metro_nodes): - if j is not (args.metro_nodes - 1): - e = ShimEthDIF("e-" + str(i) + "-" + str(j)) + if j != (args.metro_nodes - 1): + e = EthDixLayer("e-" + str(i) + "-" + str(j)) else: e = None if e_prev is None and e is not None: node = Node("metro-" + str(i) + "-" + str(j), - difs=[m, pi, e], - dif_registrations={pi: [m], m: [e]}) + layers=[m, pi, e], + registrations={pi: [m], m: [e]}) m1 = node elif e_prev is not None and e is not None: node = Node("metro-" + str(i) + "-" + str(j), - difs=[m, pi, e, e_prev], - dif_registrations={pi: [m], m: [e, e_prev]}) + layers=[m, pi, e, e_prev], + registrations={pi: [m], m: [e, e_prev]}) else: node = Node("metro-" + str(i) + "-" + str(j), - difs=[m, pi, e_prev], - dif_registrations={pi: [m], m: [e_prev]}) + layers=[m, pi, e_prev], + registrations={pi: [m], m: [e_prev]}) mn = node nodes.append(node) e_prev = e # Create nodes attached to metro for k in range(0, args.end_nodes): - ee = ShimEthDIF("e-" + str(i) + "-" + str(j) + "-" + str(k)) + ee = EthDixLayer("e-" + str(i) + "-" + str(j) + "-" + str(k)) end_node = Node("end-" + str(i) + "-" + str(j) + "-" + str(k), - difs=[pi, ee], - dif_registrations={pi: [ee]}) - node.add_dif(ee) - node.add_dif_registration(pi, ee) + layers=[pi, ee], + registrations={pi: [ee]}) + node.add_layer(ee) + node.add_registration(pi, ee) nodes.append(end_node) - e = ShimEthDIF("e-" + str(i) + "-" + str(args.metro_nodes)) - m1.add_dif(e) - m1.add_dif_registration(m, e) - mn.add_dif(e) - mn.add_dif_registration(m, e) + e = EthDixLayer("e-" + str(i) + "-" + str(args.metro_nodes)) + m1.add_layer(e) + m1.add_registration(m, e) + mn.add_layer(e) + mn.add_registration(m, e) # Add 2 links from metro to core - e0 = ShimEthDIF("c-e-" + str(i) + "-0") - e1 = ShimEthDIF("c-e-" + str(i) + "-1") - e2 = ShimEthDIF("c-e-" + str(i) + "-2") + e0 = EthDixLayer("c-e-" + str(i) + "-0") + e1 = EthDixLayer("c-e-" + str(i) + "-1") + e2 = EthDixLayer("c-e-" + str(i) + "-2") c1 = Node("c-" + str(i) + "-0", - difs=[c, m, pi, e0, e2], - dif_registrations={pi: [m, c], m: [e0], c: [e2]}) + layers=[c, m, pi, e0, e2], + registrations={pi: [m, c], m: [e0], c: [e2]}) c2 = Node("c-" + str(i) + "-1", - difs=[c, m, pi, e1, e2], - dif_registrations={pi: [m, c], m: [e1], c: [e2]}) + layers=[c, m, pi, e1, e2], + registrations={pi: [m, c], m: [e1], c: [e2]}) nodes.append(c1) nodes.append(c2) - m1.add_dif(e0) - m1.add_dif_registration(m, e0) - mn.add_dif(e1) - mn.add_dif_registration(m, e1) + m1.add_layer(e0) + m1.add_registration(m, e0) + mn.add_layer(e1) + mn.add_registration(m, e1) for x in range(0, len(core_nodes), 2): - ce0 = ShimEthDIF("c-c-e-" + str(i) + "-" + str(x)) - ce1 = ShimEthDIF("c-c-e-" + str(i) + "-" + str(x + 1)) + ce0 = EthDixLayer("c-c-e-" + str(i) + "-" + str(x)) + ce1 = EthDixLayer("c-c-e-" + str(i) + "-" + str(x + 1)) - core_nodes[x].add_dif(ce0) - core_nodes[x].add_dif_registration(c, ce0) - core_nodes[x + 1].add_dif(ce1) - core_nodes[x + 1].add_dif_registration(c, ce1) + core_nodes[x].add_layer(ce0) + core_nodes[x].add_registration(c, ce0) + core_nodes[x + 1].add_layer(ce1) + core_nodes[x + 1].add_registration(c, ce1) - c1.add_dif(ce0) - c1.add_dif_registration(c, ce0) - c2.add_dif(ce1) - c2.add_dif_registration(c, ce1) + c1.add_layer(ce0) + c1.add_registration(c, ce0) + c2.add_layer(ce1) + c2.add_registration(c, ce1) core_nodes.append(c1) core_nodes.append(c2) @@ -203,12 +185,6 @@ elif args.testbed == 'jfed': for a in jfed_p._actions if a.dest != 'help' and getattr(args, a.dest) is not None} -elif args.testbed == 'qemu': - test_class = qemu.Testbed - testbed_args = {a.dest: getattr(args, a.dest) - for a in qemu_p._actions - if a.dest != 'help' - and getattr(args, a.dest) is not None} elif args.testbed == 'local': test_class = local.Testbed testbed_args = {a.dest: getattr(args, a.dest) @@ -228,25 +204,18 @@ elif args.testbed is None: testbed = test_class(**testbed_args) -if args.prototype == 'irati': - exp_class = irati.Experiment -elif args.prototype == 'ouroboros': - exp_class = our.Experiment -elif args.prototype == 'rlite': - exp_class = rl.Experiment -else: - print('Unexpected prototype!') - print(argparser.format_help()) - exit(1) +exp_class = our.Experiment exp = exp_class(testbed, nodes=nodes) if __name__ == "__main__": - draw_experiment(exp) + + if args.visualize: + nw = get_network_from_rumba_experiment(exp) + draw_network(nw) with ExperimentManager(exp, swap_out_strategy=args.swapout) as expM: exp.swap_in() if not isinstance(testbed, docker.Testbed) \ - and not isinstance(testbed, qemu.Testbed) \ and not isinstance(testbed, local.Testbed): exp.install_prototype() exp.bootstrap_prototype() diff --git a/examples/script-example.py b/examples/script-example.py index 6cfaf42..e99ce78 100755 --- a/examples/script-example.py +++ b/examples/script-example.py @@ -6,59 +6,57 @@ from functools import partial from rumba.storyboard import * from rumba.model import * import rumba.log as log -import rumba.testbeds.qemu as qemu -import rumba.prototypes.rlite as rl +import rumba.testbeds.local as local +import rumba.prototypes.ouroboros as our from rumba.utils import ExperimentManager import rumba.utils as utils log.set_logging_level(log.DEBUG) -n1 = NormalDIF("n1") +n1 = UnicastLayer("n1") n1.add_policy("rmt.pff", "lfa") n1.add_policy("security-manager", "passwd") -e1 = ShimEthDIF("e1") +e1 = EthDixLayer("e1") node_a = Node("node_a", - difs=[n1, e1], - dif_registrations={n1: [e1]}) + layers=[n1, e1], + registrations={n1: [e1]}) node_b = Node("node_b", - difs=[e1, n1], - dif_registrations={n1: [e1]}) + layers=[e1, n1], + registrations={n1: [e1]}) -tb = qemu.Testbed(exp_name="script_test", - username="root", - password="root") +tb = local.Testbed(exp_name="script_test") -exp = rl.Experiment(tb, nodes=[node_a, node_b]) +exp = our.Experiment(tb, nodes=[node_a, node_b]) client_a = Client( - "rinaperf", - options="-t perf -s 1000 -D <duration>", + "operf", + options="-d <duration> -s 1000", shutdown="", - c_id='rinaperf_a' + c_id='operf_a' ) client_b = Client( - "rinaperf", - options="-t perf -s 1000 -D <duration> -z rinaperfb", + "operf", + options="-d <duration> -s 1000 -n operfb", shutdown="", - c_id='rinaperf_b' + c_id='operf_b' ) client_c = Client( - "rinaperf", - options="-t perf -s 1000 -D <duration> -z rinaperfc", + "operf", + options="-d <duration> -s 1000 -n operfc", shutdown="", - c_id='rinaperf_c' + c_id='operf_c' ) server_a = Server( - "rinaperf", + "operf", options="-l", arrival_rate=1, mean_duration=5, @@ -67,8 +65,8 @@ server_a = Server( ) server_b = Server( - "rinaperf", - options="-l -z rinaperfb", + "operf", + options="-l -n operfb", arrival_rate=0.5, mean_duration=10, clients=[client_b], @@ -76,8 +74,8 @@ server_b = Server( ) server_c = Server( - "rinaperf", - options="-l -z rinaperfc", + "operf", + options="-l -n operfc", arrival_rate=1.6, mean_duration=3, clients=[client_c], diff --git a/examples/snake.py b/examples/snake.py index 5426e28..ac5573f 100755 --- a/examples/snake.py +++ b/examples/snake.py @@ -3,19 +3,16 @@ # An example script using the rumba package from rumba.model import * -from rumba.utils import ExperimentManager +from rumba.utils import * +from rumba.storyboard import * +from rumba.topologies import build_chain # import testbed plugins -import rumba.testbeds.emulab as emulab import rumba.testbeds.jfed as jfed -import rumba.testbeds.local as local -import rumba.testbeds.qemu as qemu import rumba.log as log # import prototype plugins import rumba.prototypes.ouroboros as our -import rumba.prototypes.rlite as rl -import rumba.prototypes.irati as irati import argparse import sys @@ -31,40 +28,29 @@ args = argparser.parse_args() log.set_logging_level('DEBUG') -n01 = NormalDIF("n01") +n01 = UnicastLayer("n01") if (args.nodes < 2): print("Snake must be longer than 2 nodes") sys.exit(-1) -nodes = [] +nodes = build_chain(args.nodes, n01) -shim_prev = None -for i in range(0, args.nodes): - if i is not (args.nodes - 1): - shim = ShimEthDIF("e" + str(i)) - else: - shim = None +tb = jfed.Testbed(exp_name = "snake2", + cert_file = '/path/to/cert.pem', + authority = 'wall1.ilabt.iminds.be', + username = 'username', + exp_hours = '1', + proj_name='ouroborosrocks') - if shim_prev == None and shim != None: - node = Node("node" + str(i), difs = [n01, shim], - dif_registrations = {n01 : [shim]}) - elif shim_prev != None and shim != None: - node = Node("node" + str(i), difs = [n01, shim, shim_prev], - dif_registrations = {n01 : [shim, shim_prev]}) - else: - node = Node("node" + str(i), difs = [n01, shim_prev], - dif_registrations = {n01 : [shim_prev]}) - - shim_prev = shim - nodes.append(node) - -tb = qemu.Testbed(exp_name = "snake") - -exp = rl.Experiment(tb, nodes = nodes) +exp = our.Experiment(tb, nodes=nodes, + git_repo='https://codeberg.org/o7s/ouroboros', + git_branch='be', + build_options='-DCMAKE_BUILD_TYPE=Debug') print(exp) -with ExperimentManager(exp): +with ExperimentManager(exp, swap_out_strategy=PAUSE_SWAPOUT): exp.swap_in() + exp.install_prototype() exp.bootstrap_prototype() diff --git a/examples/square.py b/examples/square.py new file mode 100755 index 0000000..ce5cc41 --- /dev/null +++ b/examples/square.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python + +# An example script using the rumba package + +from rumba.model import * +from rumba.utils import ExperimentManager + +# import testbed plugins +import rumba.testbeds.local as local + +# import prototype plugins +import rumba.prototypes.ouroboros as our + +import rumba.log as log + +log.set_logging_level('DEBUG') + + +n01 = UnicastLayer("n01") +n01.add_policy("routing", "ecmp") + +e01 = EthDixLayer("e01") +e02 = EthDixLayer("e02") +e03 = EthDixLayer("e03") +e04 = EthDixLayer("e04") + + +a = Node("a", + layers = [n01, e01, e02], + registrations = {n01 : [e01, e02]}) + +b = Node("b", + layers = [n01, e02, e03], + registrations = {n01 : [e02, e03]}) + +c = Node("c", + layers = [n01, e03, e04], + registrations = {n01 : [e03, e04]}) + +d = Node("d", + layers = [n01, e01, e04], + registrations = {n01 : [e01, e04]}) + +tb = local.Testbed(exp_name = "square") + +exp = our.Experiment(tb, nodes = [a, b, c, d]) + +print(exp) + +with ExperimentManager(exp, swap_out_strategy=PROMPT_SWAPOUT): + exp.swap_in() + exp.bootstrap_prototype() diff --git a/examples/test.py b/examples/test.py new file mode 100755 index 0000000..e17670a --- /dev/null +++ b/examples/test.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python + +# An example script using the rumba package + +from rumba.model import * +from rumba.utils import ExperimentManager +from rumba.storyboard import * + + +# import testbed plugins +import rumba.testbeds.jfed as jfed + +# import prototype plugins +import rumba.prototypes.ouroboros as our + +import rumba.log as log + +log.set_logging_level('DEBUG') + + +n1 = UnicastLayer("n1") +e1 = EthDixLayer("e1") +e2 = EthDixLayer("e2") + +clientNode = Node("client", + layers = [e1, n1], + registrations = {n1 : [e1]}) + +routerNode = Node("router", + layers = [e1, e2, n1], + registrations = {n1 : [e1, e2]}) + +serverNode = Node("server", + layers = [e2, n1], + registrations = {n1 : [e2]}) + +tb = jfed.Testbed(exp_name = 'example', + cert_file = '/path/to/cert.pem', + authority = 'wall1.ilabt.iminds.be', + username = 'username', + exp_hours = '5', + proj_name='ouroboros') + +exp = our.Experiment(tb, nodes=[clientNode, routerNode, serverNode], + git_repo="https://codeberg.org/o7s/ouroboros", + git_branch="be-fix") + +with ExperimentManager(exp, swap_out_strategy=PAUSE_SWAPOUT): + exp.swap_in() + exp.install_prototype() + exp.bootstrap_prototype() + sb = StoryBoard(experiment=exp, duration=15, servers=[]) + sb.schedule_command(7.5, clientNode, + 'echo "7.5 secs in. We are at $(hostname)"') + sb.schedule_command(12, serverNode, + 'echo "12 secs in. We are at $(hostname)"') + sb.start() diff --git a/examples/two-layers.py b/examples/two-layers.py index 3f50037..5f6600c 100755 --- a/examples/two-layers.py +++ b/examples/two-layers.py @@ -6,15 +6,10 @@ from rumba.model import * from rumba.storyboard import * # import testbed plugins -import rumba.testbeds.emulab as emulab -import rumba.testbeds.jfed as jfed import rumba.testbeds.local as local -import rumba.testbeds.qemu as qemu # import prototype plugins import rumba.prototypes.ouroboros as our -import rumba.prototypes.rlite as rl -import rumba.prototypes.irati as irati import rumba.log as log from rumba.utils import ExperimentManager @@ -22,36 +17,34 @@ from rumba.utils import ExperimentManager log.set_logging_level('DEBUG') -n1 = NormalDIF("n1") -n2 = NormalDIF("n2") -n3 = NormalDIF("n3") -n4 = NormalDIF("n4") +n1 = UnicastLayer("n1") +n2 = UnicastLayer("n2") +n3 = UnicastLayer("n3") +n4 = UnicastLayer("n4") -e1 = ShimEthDIF("e1") -e2 = ShimEthDIF("e2") -e3 = ShimEthDIF("e3") +e1 = EthDixLayer("e1") +e2 = EthDixLayer("e2") +e3 = EthDixLayer("e3") a = Node("a", - difs = [n3, n4, n1, e1], - dif_registrations = {n4: [n1], n3: [n1], n1 : [e1]}) + layers = [n3, n4, n1, e1], + registrations = {n4: [n1], n3: [n1], n1 : [e1]}) b = Node("b", - difs = [n1, e1, e2], - dif_registrations = {n1 : [e1, e2]}) + layers = [n1, e1, e2], + registrations = {n1 : [e1, e2]}) c = Node("c", - difs = [n3, n4, n1, n2, e2, e3], - dif_registrations = {n4: [n1], n3: [n1, n2], n1 : [e2], n2: [e3]}) + layers = [n3, n4, n1, n2, e2, e3], + registrations = {n4: [n1], n3: [n1, n2], n1 : [e2], n2: [e3]}) d = Node("d", - difs = [n3, n2, e3], - dif_registrations = {n3: [n2], n2 : [e3]}) + layers = [n3, n2, e3], + registrations = {n3: [n2], n2 : [e3]}) -tb = qemu.Testbed(exp_name = "twolayers", - username = "root", - password = "root") +tb = local.Testbed(exp_name="twolayers") -exp = rl.Experiment(tb, nodes = [a, b, c, d]) +exp = our.Experiment(tb, nodes = [a, b, c, d]) print(exp) diff --git a/examples/vpn.py b/examples/vpn.py index 6aa8db6..8aac916 100755 --- a/examples/vpn.py +++ b/examples/vpn.py @@ -6,46 +6,39 @@ from rumba.model import * from rumba.utils import ExperimentManager # import testbed plugins -import rumba.testbeds.emulab as emulab -import rumba.testbeds.jfed as jfed import rumba.testbeds.local as local -import rumba.testbeds.qemu as qemu # import prototype plugins import rumba.prototypes.ouroboros as our -import rumba.prototypes.rlite as rl -import rumba.prototypes.irati as irati import rumba.log as log log.set_logging_level('DEBUG') -n1 = NormalDIF("n1") -n2 = NormalDIF("n2") -e1 = ShimEthDIF("e1") -e2 = ShimEthDIF("e2") -e3 = ShimEthDIF("e3") +n1 = UnicastLayer("n1") +n2 = UnicastLayer("n2") +e1 = EthDixLayer("e1") +e2 = EthDixLayer("e2") +e3 = EthDixLayer("e3") a = Node("a", - difs = [e1, n1, n2], - dif_registrations = {n1 : [e1], n2 : [n1]}) + layers = [e1, n1, n2], + registrations = {n1 : [e1], n2 : [n1]}) b = Node("b", - difs = [e1, e2, n1], - dif_registrations = {n1 : [e1, e2]}) + layers = [e1, e2, n1], + registrations = {n1 : [e1, e2]}) c = Node("c", - difs = [e2, e3, n1], - dif_registrations = {n1 : [e2, e3]}) + layers = [e2, e3, n1], + registrations = {n1 : [e2, e3]}) d = Node("d", - difs = [e3, n1, n2], - dif_registrations = {n1 : [e3], n2 : [n1]}) + layers = [e3, n1, n2], + registrations = {n1 : [e3], n2 : [n1]}) -tb = qemu.Testbed(exp_name = 'example1', - username = 'root', - password = 'root') +tb = local.Testbed(exp_name="example1") exp = our.Experiment(tb, nodes = [a, b, c, d]) diff --git a/rumba/_version.py b/rumba/_version.py index 4fb93d0..f1e5b84 100644 --- a/rumba/_version.py +++ b/rumba/_version.py @@ -24,5 +24,5 @@ # Foundation, Inc., http://www.fsf.org/about/contact/. # -__version_info__ = (0, 9) +__version_info__ = (0, 23, 0) __version__ = '.'.join(map(str, __version_info__)) diff --git a/rumba/command.py b/rumba/command.py new file mode 100644 index 0000000..0e15c4a --- /dev/null +++ b/rumba/command.py @@ -0,0 +1,60 @@ +# +# Rumba - Structured command results +# +# Copyright (C) 2017-2026 imec +# +# Dimitri Staessens <dimitri.staessens@ugent.be> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., http://www.fsf.org/about/contact/. +# + +import dataclasses + + +class CommandError(Exception): + """Raised when a command returns a non-zero exit code.""" + + def __init__(self, result): + self.result = result + msg = ("Command '%s' on node '%s' failed with exit code %d.\n" + "stdout: %s\nstderr: %s" + % (result.command, result.node, result.exit_code, + result.stdout, result.stderr)) + super().__init__(msg) + + +@dataclasses.dataclass +class CommandResult: + """Structured result of a command execution.""" + exit_code: int + stdout: str + stderr: str + command: str + node: str + + @property + def succeeded(self): + """True if the command exited with code 0.""" + return self.exit_code == 0 + + def check(self): + """Raise CommandError if exit code is non-zero. + + :returns: self (for chaining). + :raises CommandError: If exit code is non-zero. + """ + if not self.succeeded: + raise CommandError(self) + return self diff --git a/rumba/elements/experimentation.py b/rumba/elements/experimentation.py index efd7a3e..2872f7f 100644 --- a/rumba/elements/experimentation.py +++ b/rumba/elements/experimentation.py @@ -80,9 +80,9 @@ class Testbed(object): self._swap_in(experiment) - for dif in experiment.dif_ordering: - if isinstance(dif, topology.ShimEthDIF): - dif.link_quality.apply(dif) + for layer in experiment.layer_ordering: + if layer.is_eth: + layer.link_quality.apply(layer) @abc.abstractmethod def _swap_in(self, experiment): @@ -149,14 +149,14 @@ class Experiment(object): self.add_packages = add_packages self.testbed = testbed # the strategy employed for completing the enrollment phase in - # the different DIFs + # the different Layers self.enrollment_strategy = enrollment_strategy # 'full-mesh', 'manual' # the strategy employed for setting up the data transfer - # network in the DIFs after enrollment + # network in the Layers after enrollment self.flows_strategy = flows_strategy - self.dif_ordering = [] - self.enrollments = [] # a list of per-DIF lists of enrollments - self.flows = [] # a list of per-DIF lists of flows + self.layer_ordering = [] + self.enrollments = [] # a list of per-Layer lists of enrollments + self.flows = [] # a list of per-Layer lists of flows if self.enrollment_strategy not in ['full-mesh', 'minimal', 'manual']: raise Exception('Unknown enrollment strategy "%s"' @@ -209,120 +209,120 @@ class Experiment(object): self.nodes.remove(node) self.generate() - # Compute registration/enrollment order for DIFs - def compute_dif_ordering(self): - # Compute DIFs dependency graph, as both adjacency and incidence list. - difsdeps_adj = dict() - difsdeps_inc = dict() + # Compute registration/enrollment order for Layers + def compute_layer_ordering(self): + # Compute Layers dependency graph, as both adjacency and incidence list. + layerdeps_adj = dict() + layerdeps_inc = dict() for node in self.nodes: - for dif in node.difs: - if dif not in difsdeps_adj: - difsdeps_adj[dif] = set() - - for upper in node.dif_registrations: - for lower in node.dif_registrations[upper]: - if upper not in difsdeps_inc: - difsdeps_inc[upper] = set() - if lower not in difsdeps_inc: - difsdeps_inc[lower] = set() - if upper not in difsdeps_adj: - difsdeps_adj[upper] = set() - if lower not in difsdeps_adj: - difsdeps_adj[lower] = set() - difsdeps_inc[upper].add(lower) - difsdeps_adj[lower].add(upper) + for layer in node.layers: + if layer not in layerdeps_adj: + layerdeps_adj[layer] = set() + + for upper in node.registrations: + for lower in node.registrations[upper]: + if upper not in layerdeps_inc: + layerdeps_inc[upper] = set() + if lower not in layerdeps_inc: + layerdeps_inc[lower] = set() + if upper not in layerdeps_adj: + layerdeps_adj[upper] = set() + if lower not in layerdeps_adj: + layerdeps_adj[lower] = set() + layerdeps_inc[upper].add(lower) + layerdeps_adj[lower].add(upper) # Kahn's algorithm below only needs per-node count of # incident edges, so we compute these counts from the # incidence list and drop the latter. - difsdeps_inc_cnt = dict() - for dif in difsdeps_inc: - difsdeps_inc_cnt[dif] = len(difsdeps_inc[dif]) - del difsdeps_inc + layerdeps_inc_cnt = dict() + for layer in layerdeps_inc: + layerdeps_inc_cnt[layer] = len(layerdeps_inc[layer]) + del layerdeps_inc - # Init difsdeps_inc_cnt for those DIFs that do not + # Init layerdeps_inc_cnt for those Layers that do not # act as lower IPCPs nor upper IPCPs for registration # operations for node in self.nodes: - for dif in node.difs: - if dif not in difsdeps_inc_cnt: - difsdeps_inc_cnt[dif] = 0 + for layer in node.layers: + if layer not in layerdeps_inc_cnt: + layerdeps_inc_cnt[layer] = 0 # Run Kahn's algorithm to compute topological - # ordering on the DIFs graph. + # ordering on the Layers graph. frontier = set() - self.dif_ordering = [] - for dif in difsdeps_inc_cnt: - if difsdeps_inc_cnt[dif] == 0: - frontier.add(dif) + self.layer_ordering = [] + for layer in layerdeps_inc_cnt: + if layerdeps_inc_cnt[layer] == 0: + frontier.add(layer) while len(frontier): cur = frontier.pop() - self.dif_ordering.append(cur) - for nxt in difsdeps_adj[cur]: - difsdeps_inc_cnt[nxt] -= 1 - if difsdeps_inc_cnt[nxt] == 0: + self.layer_ordering.append(cur) + for nxt in layerdeps_adj[cur]: + layerdeps_inc_cnt[nxt] -= 1 + if layerdeps_inc_cnt[nxt] == 0: frontier.add(nxt) - difsdeps_adj[cur] = set() + layerdeps_adj[cur] = set() - circular_set = [dif for dif in difsdeps_inc_cnt - if difsdeps_inc_cnt[dif] != 0] + circular_set = [layer for layer in layerdeps_inc_cnt + if layerdeps_inc_cnt[layer] != 0] if len(circular_set): - raise Exception("Fatal error: The specified DIFs topology" + raise Exception("Fatal error: The specified Layers topology" "has one or more" "circular dependencies, involving the following" - " DIFs: %s" % circular_set) + " Layers: %s" % circular_set) - logger.debug("DIF topological ordering: %s", self.dif_ordering) + logger.debug("Layer topological ordering: %s", self.layer_ordering) - # Compute all the enrollments, to be called after compute_dif_ordering() + # Compute all the enrollments, to be called after compute_layer_ordering() def compute_enrollments(self): - dif_graphs = dict() + layer_graphs = dict() self.enrollments = [] self.flows = [] - for dif in self.dif_ordering: + for layer in self.layer_ordering: neighsets = dict() - dif_graphs[dif] = dict() + layer_graphs[layer] = dict() first = None - # For each N-1-DIF supporting this DIF, compute the set of nodes - # that share such N-1-DIF. This set will be called the 'neighset' of - # the N-1-DIF for the current DIF. + # For each N-1-Layer supporting this Layer, compute the set of nodes + # that share such N-1-Layer. This set will be called the 'neighset' of + # the N-1-Layer for the current Layer. for node in self.nodes: - if dif in node.dif_registrations: - dif_graphs[dif][node] = [] # init for later use + if layer in node.registrations: + layer_graphs[layer][node] = [] # init for later use if first is None: # pick any node for later use first = node - for lower_dif in node.dif_registrations[dif]: - if lower_dif not in neighsets: - neighsets[lower_dif] = [] - neighsets[lower_dif].append(node) + for lower_layer in node.registrations[layer]: + if lower_layer not in neighsets: + neighsets[lower_layer] = [] + neighsets[lower_layer].append(node) # Build the graph, represented as adjacency list - for lower_dif in neighsets: + for lower_layer in neighsets: # Each neighset corresponds to a complete (sub)graph. - for node1 in neighsets[lower_dif]: - for node2 in neighsets[lower_dif]: + for node1 in neighsets[lower_layer]: + for node2 in neighsets[lower_layer]: if node1 != node2: - dif_graphs[dif][node1].append((node2, lower_dif)) + layer_graphs[layer][node1].append((node2, lower_layer)) self.enrollments.append([]) self.flows.append([]) if first is None: - # This is a shim DIF, nothing to do + # This is a shim Layer, nothing to do continue er = [] - for node in dif_graphs[dif]: - for edge in dif_graphs[dif][node]: + for node in layer_graphs[layer]: + for edge in layer_graphs[layer][node]: er.append("%s --[%s]--> %s" % (node.name, edge[1].name, edge[0].name)) - logger.debug("DIF graph for %s: %s", dif, ', '.join(er)) + logger.debug("Layer graph for %s: %s", layer, ', '.join(er)) # To generate the list of minimal enrollments # and minimal flows, we simulate it, using @@ -332,39 +332,39 @@ class Experiment(object): edges_covered = set() while len(frontier): cur = frontier.pop() - for edge in dif_graphs[dif][cur]: + for edge in layer_graphs[layer][cur]: if edge[0] not in enrolled: enrolled.add(edge[0]) - enrollee = edge[0].get_ipcp_by_dif(dif) + enrollee = edge[0].get_ipcp_by_layer(layer) assert(enrollee is not None) - enroller = cur.get_ipcp_by_dif(dif) + enroller = cur.get_ipcp_by_layer(layer) assert(enroller is not None) edges_covered.add((enrollee, enroller)) - self.enrollments[-1].append({'dif': dif, + self.enrollments[-1].append({'layer': layer, 'enrollee': enrollee, 'enroller': enroller, - 'lower_dif': edge[1]}) + 'lower_layer': edge[1]}) self.flows[-1].append({'src': enrollee, 'dst': enroller}) frontier.add(edge[0]) - if len(dif.members) != len(enrolled): - raise Exception("Disconnected DIF found: %s" % (dif,)) + if len(layer.members) != len(enrolled): + raise Exception("Disconnected Layer found: %s" % (layer,)) # In case of a full mesh enrollment or dt flows - for cur in dif_graphs[dif]: - for edge in dif_graphs[dif][cur]: + for cur in layer_graphs[layer]: + for edge in layer_graphs[layer][cur]: if cur.name < edge[0].name: - enrollee = cur.get_ipcp_by_dif(dif) + enrollee = cur.get_ipcp_by_layer(layer) assert(enrollee is not None) - enroller = edge[0].get_ipcp_by_dif(dif) + enroller = edge[0].get_ipcp_by_layer(layer) assert(enroller is not None) if ((enrollee, enroller) not in edges_covered and (enroller, enrollee) not in edges_covered): if self.enrollment_strategy == 'full-mesh': - self.enrollments[-1].append({'dif': dif, + self.enrollments[-1].append({'layer': layer, 'enrollee': enrollee, 'enroller': enroller, - 'lower_dif': edge[1]}) + 'lower_layer': edge[1]}) if self.flows_strategy == 'full-mesh': self.flows[-1].append({'src': enrollee, 'dst': enroller}) @@ -380,11 +380,11 @@ class Experiment(object): log_string = "Enrollments:\n" for el in self.enrollments: for e in el: - log_string += (" [%s] %s --> %s through N-1-DIF %s\n" - % (e['dif'], + log_string += (" [%s] %s --> %s through N-1-Layer %s\n" + % (e['layer'], e['enrollee'].name, e['enroller'].name, - e['lower_dif'])) + e['lower_layer'])) logger.debug(log_string) log_string = "Flows:\n" @@ -402,40 +402,41 @@ class Experiment(object): node.ipcps = [] # We want also the node.ipcps list to be generated in # topological ordering - for dif in self.dif_ordering: - if dif not in node.difs: + for layer in self.layer_ordering: + if layer not in node.layers: continue # Create an instance of the required IPCP class - ipcp = dif.get_ipcp_class()( - name='%s.%s' % (dif.name, node.name), - node=node, dif=dif) + ipcp = layer.get_ipcp_class()( + name='%s.%s' % (layer.name, node.name), + node=node, layer=layer) - if dif in node.dif_registrations: - for lower in node.dif_registrations[dif]: + if layer in node.registrations: + for lower in node.registrations[layer]: ipcp.registrations.append(lower) node.ipcps.append(ipcp) - dif.ipcps.append(ipcp) + layer.ipcps.append(ipcp) def compute_bootstrappers(self): for node in self.nodes: for ipcp in node.ipcps: - ipcp.dif_bootstrapper = True + ipcp.bootstrapper = True for el in self.enrollments: for e in el: - if e['dif'] != ipcp.dif: - # Skip this DIF + if e['layer'] != ipcp.layer: + # Skip this Layer break if e['enrollee'] == ipcp: - ipcp.dif_bootstrapper = False + ipcp.bootstrapper = False # Exit the loops break - if not ipcp.dif_bootstrapper: + if not ipcp.bootstrapper: break def dump_ssh_info(self): f = open(os.path.join(tmp_dir, 'ssh_info'), 'w') + for node in self.nodes: f.write("%s;%s;%s;%s;%s\n" % (node.name, self.testbed.username, @@ -444,11 +445,11 @@ class Experiment(object): node.ssh_config.proxy_server)) f.close() - # Examine the nodes and DIFs, compute the registration and enrollment + # Examine the nodes and Layers, compute the registration and enrollment # order, the list of IPCPs to create, registrations, ... def generate(self): start = time.time() - self.compute_dif_ordering() + self.compute_layer_ordering() self.compute_ipcps() self.compute_enrollments() self.compute_bootstrappers() @@ -509,8 +510,8 @@ class Experiment(object): logger.info("Stopping metrics exporters took %.2f seconds", end - start) @abc.abstractmethod - def destroy_dif(self, dif): - raise Exception('destroy_dif() method not implemented') + def destroy_layer(self, layer): + raise Exception('destroy_layer() method not implemented') @abc.abstractmethod def _install_prototype(self): @@ -596,24 +597,8 @@ class Experiment(object): node.execute_command('reboot', as_root=True) @abc.abstractmethod - def export_dif_bandwidth(self, filename, dif): - raise Exception('Export DIF bandwidth method not implemented') - - def to_dms_yaml(self, filename): - """ - Generate a YAML file of the experiment which can be fed to the - ARCFIRE DMS. - - :param filename: The output YAML filename. - """ - mode = 'w' - with open(filename, mode) as f: - for node in self.nodes: - f.write("---\n") - node.to_dms_yaml(f) - f.write("...\n") - - logger.info('Generated DMS YAML file') + def export_layer_bandwidth(self, filename, layer): + raise Exception('Export Layer bandwidth method not implemented') def export_connectivity_graph(self, filename): """ @@ -647,11 +632,11 @@ class Experiment(object): return try: - for dif in self.dif_ordering: - if isinstance(dif, topology.ShimEthDIF): - edge = pydot.Edge(dif.members[0].name, - dif.members[1].name, - label=dif.name, + for layer in self.layer_ordering: + if layer.is_eth: + edge = pydot.Edge(layer.members[0].name, + layer.members[1].name, + label=layer.name, color='black') gvizg.add_edge(edge) except Exception as e: @@ -668,12 +653,12 @@ class Experiment(object): logger.error('Warning: pydot module not installed, ' 'cannot produce graph images') - def export_dif_graph(self, filename, dif): + def export_layer_graph(self, filename, layer): """ - Generate a PDF of a DIF graph. + Generate a PDF of a Layer graph. :param filename: The output PDF filename. - :param dif: The DIF to export. + :param layer: The Layer to export. """ try: import pydot @@ -685,7 +670,7 @@ class Experiment(object): i = 0 try: - for node in dif.members: + for node in layer.members: g_node = pydot.Node(node.name, label=node.name, style="filled", @@ -704,7 +689,7 @@ class Experiment(object): for enroll, dt in zip(self.enrollments, self.flows): for e in enroll: - if e['dif'] is not dif: + if e['layer'] is not layer: continue edge = pydot.Edge(e['enrollee'].node.name, @@ -713,7 +698,7 @@ class Experiment(object): gvizg.add_edge(edge) for e in dt: - if e['src'].dif is not dif: + if e['src'].layer is not layer: continue edge = pydot.Edge(e['src'].node.name, @@ -726,21 +711,30 @@ class Experiment(object): try: gvizg.write_pdf(filename) - logger.info('Generated PDF of DIF graph') + logger.info('Generated PDF of Layer graph') except Exception as e: logger.error('Failed to write PDF: ' + str(e)) except: logger.error('Warning: pydot module not installed, ' - 'cannot produce DIF graph images') + 'cannot produce Layer graph images') class Executor: __metaclass__ = abc.ABCMeta @abc.abstractmethod - def execute_command(self, node, command, as_root=False, time_out=3): - # Execute command on a node + def execute_command(self, node, command, as_root=False, time_out=3, + check=True): + """Execute command on a node. + + :param node: The node to execute on. + :param command: The command string. + :param as_root: Execute as root. + :param time_out: Timeout in seconds. + :param check: If True, raise CommandError on non-zero exit. + :returns: A CommandResult. + """ return def execute_commands(self, node, commands, as_root=False, time_out=3): diff --git a/rumba/elements/topology.py b/rumba/elements/topology.py index 5a3ad17..1e2c1a6 100644 --- a/rumba/elements/topology.py +++ b/rumba/elements/topology.py @@ -1,13 +1,10 @@ # -# A library to manage ARCFIRE experiments +# Rumba - Topology Model # -# Copyright (C) 2017-2018 Nextworks S.r.l. -# Copyright (C) 2017-2018 imec +# Copyright (C) 2017-2026 imec # # Sander Vrijders <sander.vrijders@ugent.be> # Dimitri Staessens <dimitri.staessens@ugent.be> -# Vincenzo Maffione <v.maffione@nextworks.it> -# Marco Capitani <m.capitani@nextworks.it> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -31,27 +28,50 @@ import rumba.log as log logger = log.get_logger(__name__) -class DIF(object): +# --------------------------------------------------------------------------- +# Layer types (aligned with Ouroboros IPCP types) +# --------------------------------------------------------------------------- + +class LayerType(Enum): + """IPCP / layer types available in Ouroboros.""" + LOCAL = 'local' + UNICAST = 'unicast' + BROADCAST = 'broadcast' + ETH_LLC = 'eth-llc' + ETH_DIX = 'eth-dix' + UDP4 = 'udp4' + UDP6 = 'udp6' + + +# --------------------------------------------------------------------------- +# Layer classes +# --------------------------------------------------------------------------- + +class Layer(object): """ - Base class for DIFs. + Base class for network layers. + + A Layer represents a network layer in an Ouroboros network. """ def get_e_id(self): - return "DIF." + self.name + return "Layer." + self.name - def __init__(self, name, members=None): + def __init__(self, name, layer_type=LayerType.UNICAST, members=None): """ - :param name: Name of the DIF. - :param members: List of nodes that are members of the DIF. + :param name: Name of the layer. + :param layer_type: Type of the layer (LayerType enum). + :param members: List of nodes that are members of this layer. """ self.name = name + self.layer_type = layer_type if members is None: members = list() self.members = members self.ipcps = list() def __repr__(self): - s = "DIF %s" % self.name + s = "Layer %s (%s)" % (self.name, self.layer_type.value) return s def __hash__(self): @@ -72,49 +92,79 @@ class DIF(object): def get_ipcp_class(self): return IPCP + @property + def is_eth(self): + return self.layer_type in (LayerType.ETH_DIX, LayerType.ETH_LLC) -# Shim over UDP -# -class ShimUDPDIF(DIF): + @property + def is_udp(self): + return self.layer_type in (LayerType.UDP4, LayerType.UDP6) + + @property + def is_shim(self): + return self.layer_type in (LayerType.ETH_DIX, LayerType.ETH_LLC, + LayerType.UDP4, LayerType.UDP6, + LayerType.LOCAL) + + +# Shim over UDP (IPv4) +class Udp4Layer(Layer): """ - Shim over UDP. + Layer using the UDP over IPv4 shim IPCP. """ def __init__(self, name, members=None): """ - :param name: Name of the DIF. - :param members: List of members of the DIF. + :param name: Name of the layer. + :param members: List of members of the layer. """ - DIF.__init__(self, name, members) + Layer.__init__(self, name, LayerType.UDP4, members) def get_ipcp_class(self): - return ShimUDPIPCP + return UdpIPCP -# Shim over Ethernet -# -# @link_speed [int] Speed of the Ethernet network, in Mbps -# -class ShimEthDIF(DIF): +# Shim over UDP (IPv6) +class Udp6Layer(Layer): + """ + Layer using the UDP over IPv6 shim IPCP. + """ + def __init__(self, name, members=None): + """ + :param name: Name of the layer. + :param members: List of members of the layer. + """ + Layer.__init__(self, name, LayerType.UDP6, members) + + def get_ipcp_class(self): + return UdpIPCP + + +# Ethernet layer (point-to-point link) +class EthDixLayer(Layer): """ - Shim over Ethernet. + Layer using an Ethernet shim IPCP (eth-dix). + Models a point-to-point Ethernet link between two nodes. """ def get_e_id(self): - return "ShimEthDIF." + self.name + return "EthDixLayer." + self.name - def __init__(self, name, members=None, link_quality=None): + def __init__(self, name, members=None, link_quality=None, + layer_type=LayerType.ETH_DIX): """ - :param name: Name of the DIF. - :param members: List of members of the DIF. + :param name: Name of the layer. + :param members: List of members (max 2 for point-to-point). :param link_quality: Quality of the link. + :param layer_type: LayerType (ETH_DIX or ETH_LLC). """ - DIF.__init__(self, name, members) - self._link_quality = link_quality if link_quality is not None else LinkQuality() + Layer.__init__(self, name, layer_type, members) + self._link_quality = link_quality if link_quality is not None \ + else LinkQuality() def get_ipcp_class(self): - return ShimEthIPCP + return EthIPCP def add_member(self, node): - super(ShimEthDIF, self).add_member(node) + super(EthDixLayer, self).add_member(node) if len(self.members) > 2: raise Exception("More than 2 members in %s!" % self.name) @@ -127,9 +177,7 @@ class ShimEthDIF(DIF): if not _link_quality: raise ValueError("Cannot set link_quality to None, use del " "link_quality to reset") - self._link_quality = _link_quality - _link_quality.apply(self) @link_quality.deleter @@ -150,8 +198,7 @@ class ShimEthDIF(DIF): :type jitter: :py:class:`int` :param correlation: correlation in % :type correlation: :py:class:`int` - :param distribution: delay distribution, defaults to a Normal - distribution + :param distribution: delay distribution, defaults to Normal :type distribution: :py:class:`.Distribution` """ new_delay = Delay(delay, jitter, correlation, distribution) @@ -186,8 +233,7 @@ class ShimEthDIF(DIF): def set_quality(self, delay, loss, rate): """ - Configure the basic quality parameters of the - underlying link. + Configure the basic quality parameters of the underlying link. :param delay: the link delay, in ms :type delay: :py:class:`int` @@ -200,9 +246,26 @@ class ShimEthDIF(DIF): self.link_quality = new_quality -class NormalDIF(DIF): +class EthLlcLayer(EthDixLayer): """ - Normal DIF. + Layer using an Ethernet shim IPCP (eth-llc). + Models a point-to-point Ethernet link between two nodes. + """ + def get_e_id(self): + return "EthLlcLayer." + self.name + + def __init__(self, name, members=None, link_quality=None): + """ + :param name: Name of the layer. + :param members: List of members (max 2 for point-to-point). + :param link_quality: Quality of the link. + """ + super().__init__(name, members, link_quality, LayerType.ETH_LLC) + + +class UnicastLayer(Layer): + """ + Unicast network layer. """ def __init__(self, name, @@ -210,13 +273,13 @@ class NormalDIF(DIF): policy=None, add_default_qos_cubes=True): """ - :param name: The name of the DIF. + :param name: The name of the layer. :param members: The list of members. - :param policy: Policies of the normal DIF. - :param add_default_qos_cubes: should the prototype-dependant default - QoS cubes be added to this DIF? + :param policy: Policies of the layer. + :param add_default_qos_cubes: should prototype-default QoS cubes + be added? """ - DIF.__init__(self, name, members) + Layer.__init__(self, name, LayerType.UNICAST, members) if policy is None: policy = Policy(self) self.policy = policy @@ -226,7 +289,7 @@ class NormalDIF(DIF): def add_policy(self, comp, pol, **params): """ - Adds a policy to the DIF. + Adds a policy to the layer. :param comp: Component name. :param pol: Policy name @@ -236,7 +299,7 @@ class NormalDIF(DIF): def del_policy(self, comp=None, pol=None): """ - Deletes a policy from the DIF. + Deletes a policy from the layer. :param comp: Component name. :param pol: Policy name @@ -245,9 +308,9 @@ class NormalDIF(DIF): def show(self): """ - :return: A string representing the policies in the DIF. + :return: A string representing the policies in the layer. """ - s = DIF.__repr__(self) + s = Layer.__repr__(self) for comp, pol_dict in self.policy.get_policies().items(): for pol, params in pol_dict.items(): s += "\n Component %s has policy %s with params %s" \ @@ -256,7 +319,7 @@ class NormalDIF(DIF): def add_qos_cube(self, name, **kwargs): """ - Adds a QoS Cube to this DIF + Adds a QoS Cube to this layer. :param name: the name to be assigned to the QoS cube :type name: `str` @@ -271,12 +334,11 @@ class NormalDIF(DIF): def del_qos_cube(self, name, strict=True): """ - Deletes a QoS cube from this DIF + Deletes a QoS cube from this layer. :param name: the name of the cube to delete :type name: `str` - :param strict: if no cube with the provided name exists, - raise an exception if and only if `strict` is `True` + :param strict: if True, raise if cube not found :type strict: `bool` """ for i, cube in enumerate(self.qos_cubes): @@ -285,13 +347,41 @@ class NormalDIF(DIF): break else: # no match if strict: - raise ValueError("No cube with name %s found in dif %s" + raise ValueError("No cube with name %s found in layer %s" % (name, self.name)) else: return self.qos_cubes.pop(index) +class BroadcastLayer(Layer): + """ + Broadcast network layer. + """ + def __init__(self, name, members=None): + """ + :param name: The name of the layer. + :param members: The list of members. + """ + Layer.__init__(self, name, LayerType.BROADCAST, members) + + +class LocalLayer(Layer): + """ + Local layer (for local loopback testing). + """ + def __init__(self, name, members=None): + """ + :param name: The name of the layer. + :param members: The list of members. + """ + Layer.__init__(self, name, LayerType.LOCAL, members) + + +# --------------------------------------------------------------------------- +# Link quality classes +# --------------------------------------------------------------------------- + class Distribution(Enum): """ An enum holding different statistical distributions. @@ -324,23 +414,17 @@ class Delay(object): :type jitter: :py:class:`int` :param correlation: correlation in % :type correlation: :py:class:`int` - :param distribution: delay distribution, defaults to a Normal - distribution + :param distribution: delay distribution, defaults to Normal :type distribution: :py:class:`.Distribution` """ - if delay < 0: raise ValueError("Delay needs to be at least 0") - if jitter and not jitter > 0: raise ValueError("Jitter needs to be higher than 0") - if (not jitter) and correlation: raise ValueError("Correlation requires a value for jitter") - if correlation and (correlation < 0 or correlation > 100): raise ValueError("Correlation needs to be between 0 and 100") - self._delay = delay self._jitter = jitter self._correlation = correlation @@ -364,16 +448,12 @@ class Delay(object): def build_command(self): opts = ["delay %ims" % self.delay] - if self.jitter: opts.append("%ims" % self.jitter) - if self.correlation: opts.append("%f%%" % self.correlation) - if self.distribution: opts.append("distribution %s" % self.distribution.name.lower()) - return " ".join(opts) @@ -392,10 +472,8 @@ class Loss(object): """ if loss and (loss < 0 or loss > 100): raise ValueError("Loss needs to be between 0 and 100") - if correlation and (correlation < 0 or correlation > 100): raise ValueError("Correlation needs to be between 0 and 100") - self._loss = loss self._correlation = correlation @@ -409,10 +487,8 @@ class Loss(object): def build_command(self): opts = ["loss %f%%" % self.loss] - if self.correlation: opts.append("%f%%" % self.correlation) - return " ".join(opts) @@ -425,22 +501,13 @@ class LinkQuality(object): @classmethod def clone(cls, old_quality, delay=None, loss=None, rate=None): """ - Clone old_quality, updating it with the provided parameters - if present. - - :param old_quality: A :py:class:`.LinkQuality` instance to - use as a base - :type old_quality: :py:class:`.LinkQuality` - :param delay: Delay object holding delay configuration - or number corresponding to delay in ms - :type delay: :py:class:`.Delay` or :py:class:`int` - :param loss: Loss object holding delay configuration or - number corresponding to loss percentage - :type loss: :py:class:`.Loss` or :py:class:`float` - :param rate: The rate of the link in mbit - :type rate: :py:class:`int` + Clone old_quality, updating with provided parameters. + + :param old_quality: A :py:class:`.LinkQuality` instance + :param delay: Delay object or number (ms) + :param loss: Loss object or number (%) + :param rate: Rate in mbit :return: a new :py:class:`.LinkQuality` instance. - :rtype: :py:class:`.LinkQuality` """ if delay is None: delay = old_quality.delay @@ -454,19 +521,12 @@ class LinkQuality(object): """ Link quality configuration. - :param delay: Delay object holding delay configuration - or number corresponding to delay in ms - :type delay: :py:class:`.Delay` or :py:class:`int` - :param loss: Loss object holding delay configuration or - number corresponding to loss percentage - :type loss: :py:class:`.Loss` or :py:class:`float` - :param rate: The rate of the link in mbit - :type rate: :py:class:`int` + :param delay: Delay object or number (ms) + :param loss: Loss object or number (%) + :param rate: Rate in mbit """ - if rate and not rate > 0: raise ValueError("Rate needs to be higher than 0") - if isinstance(delay, int) or isinstance(delay, float): delay = Delay(delay) if isinstance(loss, int) or isinstance(loss, float): @@ -496,15 +556,18 @@ class LinkQuality(object): cmds.append("tc qdisc del dev %s root" % ipcp.ifname) if self.rate: - cmds.append("tc qdisc add dev %s root handle 1: htb default 1" \ - % ipcp.ifname) - cmds.append("tc class add dev %s parent 1: classid 1:1 htb rate %imbit" \ - % (ipcp.ifname, self.rate)) + cmds.append( + "tc qdisc add dev %s root handle 1: htb default 1" + % ipcp.ifname) + cmds.append( + "tc class add dev %s parent 1: classid 1:1 htb rate %imbit" + % (ipcp.ifname, self.rate)) qref = "parent 1:1" if self.delay or self.loss: - netem_cmd.append("tc qdisc add dev %s %s netem" \ - % (ipcp.ifname, qref)) + netem_cmd.append( + "tc qdisc add dev %s %s netem" + % (ipcp.ifname, qref)) if self.delay: netem_cmd.append(self.delay.build_command()) if self.loss: @@ -513,34 +576,39 @@ class LinkQuality(object): return cmds - def apply(self, shim): + def apply(self, layer): if not (self.delay or self.loss or self.rate): - self.deactivate(shim) + self.deactivate(layer) else: - for ipcp in shim.ipcps: + for ipcp in layer.ipcps: if not ipcp.ifname: - logger.error("Could not apply LinkQuality to IPCP because " - "the interface name is None") + logger.error( + "Could not apply LinkQuality to IPCP because " + "the interface name is None") continue - - ipcp.node.execute_commands(self.build_commands(ipcp), - as_root=True) + ipcp.node.execute_commands( + self.build_commands(ipcp), as_root=True) LinkQuality._active.add(ipcp) - def deactivate(self, shim): - for ipcp in shim.ipcps: + def deactivate(self, layer): + for ipcp in layer.ipcps: if ipcp not in LinkQuality._active: continue if not ipcp.ifname: - logger.error("Could not remove LinkQuality from IPCP because " - "the interface name is None") + logger.error( + "Could not remove LinkQuality from IPCP because " + "the interface name is None") continue - - ipcp.node.execute_command("tc qdisc del dev %s root" - % ipcp.ifname, as_root=True) + ipcp.node.execute_command( + "tc qdisc del dev %s root" % ipcp.ifname, + as_root=True) LinkQuality._active.remove(ipcp) +# --------------------------------------------------------------------------- +# SSH configuration +# --------------------------------------------------------------------------- + class SSHConfig(object): def __init__(self, hostname, port=22, proxy_server=None): self.username = None @@ -558,31 +626,42 @@ class SSHConfig(object): self.password = password +# --------------------------------------------------------------------------- +# Node +# --------------------------------------------------------------------------- + class Node(object): """ - A node in the experiment. + A node (system) in the experiment. + + Nodes declare their layer memberships and registrations:: + + a = Node("a", + layers=[n1, e1], + registrations={n1: [e1]}) """ + def get_e_id(self): return "Node." + self.name - def __init__(self, name, difs=None, dif_registrations=None, + def __init__(self, name, layers=None, registrations=None, policies=None, machine_type=None): """ :param name: Name of the node. - :param difs: A list of DIFs the node is in. - :param dif_registrations: How the DIFs are stacked. - :param policies: Policies of a DIF specific to the node. - :param machine_type: Type of machine to use, physical or virtual. + :param layers: A list of layers the node is a member of. + :param registrations: How layers are stacked (upper -> [lower]). + :param policies: Policies specific to this node. + :param machine_type: Type of machine (physical or virtual). """ self.name = name - if difs is None: - difs = list() - self.difs = difs - for dif in self.difs: - dif.add_member(self) - if dif_registrations is None: - dif_registrations = dict() - self.dif_registrations = dif_registrations + if layers is None: + layers = list() + self.layers = layers + for layer in self.layers: + layer.add_member(self) + if registrations is None: + registrations = dict() + self.registrations = registrations self.machine_type = machine_type self.ssh_config = SSHConfig(name) self.ipcps = [] @@ -590,50 +669,49 @@ class Node(object): self.has_tcpdump = False if policies is None: policies = dict() - for dif in self.difs: - if hasattr(dif, 'policy'): # check if the dif supports policies - self.policies[dif] = policies.get(dif, Policy(dif, self)) + for layer in self.layers: + if hasattr(layer, 'policy'): + self.policies[layer] = policies.get(layer, Policy(layer, self)) - self.executor = None # will be set by testbed on swap_in - self.startup_command = None # will be set by prototype + self.executor = None # set by testbed on swap_in + self.startup_command = None # set by prototype self._validate() - def get_ipcp_by_dif(self, dif): + def get_ipcp_by_layer(self, layer): """ - :param dif: The DIF to get the IPCP of. - :return: The IPCP of the node that is in the DIF. + :param layer: The layer to get the IPCP of. + :return: The IPCP of the node that is in the layer. """ for ipcp in self.ipcps: - if ipcp.dif == dif: + if ipcp.layer == layer: return ipcp - def _undeclared_dif(self, dif): - if dif not in self.difs: + def _undeclared_layer(self, layer): + if layer not in self.layers: raise Exception("Invalid registration: node %s is not declared " - "to be part of DIF %s" % (self.name, dif.name)) + "to be part of layer %s" + % (self.name, layer.name)) def _validate(self): - # Check that DIFs referenced in self.dif_registrations - # are part of self.difs - for upper in self.dif_registrations: - self._undeclared_dif(upper) - for lower in self.dif_registrations[upper]: - self._undeclared_dif(lower) + for upper in self.registrations: + self._undeclared_layer(upper) + for lower in self.registrations[upper]: + self._undeclared_layer(lower) def __repr__(self): s = "Node " + self.name + ":\n" - s += " DIFs: [ " - s += " ".join([d.name for d in self.difs]) + s += " Layers: [ " + s += " ".join([l.name for l in self.layers]) s += " ]\n" - s += " DIF registrations: [ " + s += " Registrations: [ " rl = [] - for upper in self.dif_registrations: - difs = self.dif_registrations[upper] + for upper in self.registrations: + lowers = self.registrations[upper] x = "%s => [" % upper.name - x += " ".join([lower.name for lower in difs]) + x += " ".join([lower.name for lower in lowers]) x += "]" rl.append(x) s += ", ".join(rl) @@ -650,224 +728,189 @@ class Node(object): def __neq__(self, other): return not self == other - def to_dms_yaml(self, buffer): - buffer.write("node: %s\n" % (self.name)) - buffer.write("registrations:\n") - - for dif in self.dif_registrations: - if isinstance(dif, NormalDIF): - buffer.write(" - %s:\n" % (dif.name)) - for reg in self.dif_registrations[dif]: - tp = "normal" - nm = reg.name - if isinstance(reg, ShimEthDIF): - tp = "eth" - for member in reg.members: - if member.name is not self.name: - nm = member.name - buffer.write(" - %sdifid: %s, diftype: %s, " - "diftype_number: 1%s\n" % ("{", nm, tp, "}")) - - def add_dif(self, dif): - """ - Adds a DIF to the list. - - :param dif: Name of the DIF to add. - """ - self.difs.append(dif) - dif.add_member(self) - if hasattr(dif, 'policy'): - self.policies[dif] = Policy(dif, self) + def add_layer(self, layer): + """ + Adds a layer to this node. + + :param layer: The layer to add. + """ + self.layers.append(layer) + layer.add_member(self) + if hasattr(layer, 'policy'): + self.policies[layer] = Policy(layer, self) self._validate() - def del_dif(self, dif): + def del_layer(self, layer): """ - Adds a DIF to the list. + Removes a layer from this node. - :param dif: Name of the DIF to add. + :param layer: The layer to remove. """ - self.difs.remove(dif) - dif.del_member(self) + self.layers.remove(layer) + layer.del_member(self) try: - del self.policies[dif] + del self.policies[layer] except KeyError: - # It was not in there, so nothing to do pass self._validate() - def add_dif_registration(self, upper, lower): + def add_registration(self, upper, lower): """ - Adds a DIF registration. + Adds a layer registration. - :param upper: Name of the DIF that is requesting IPC. - :param lower: Name of the DIF providing IPC. + :param upper: The upper layer requesting IPC. + :param lower: The lower layer providing IPC. """ - self.dif_registrations[upper].append(lower) + if upper not in self.registrations: + self.registrations[upper] = [] + self.registrations[upper].append(lower) self._validate() - def del_dif_registration(self, upper, lower): + def del_registration(self, upper, lower): """ - Removes a DIF registration. + Removes a layer registration. - :param upper: Name of the DIF that is requesting IPC. - :param lower: Name of the DIF providing IPC. + :param upper: The upper layer. + :param lower: The lower layer. """ - self.dif_registrations[upper].remove(lower) + self.registrations[upper].remove(lower) self._validate() - def add_policy(self, dif, component_name, policy_name, **parameters): + def add_policy(self, layer, component_name, policy_name, **parameters): """ Adds a policy. - :param dif: The name of the DIF. + :param layer: The layer. :param component_name: Name of the component. :param policy_name: Name of the policy. :param parameters: Parameters of the policy. """ - self.policies[dif].add_policy(component_name, policy_name, **parameters) + self.policies[layer].add_policy( + component_name, policy_name, **parameters) - def del_policy(self, dif, component_name=None, policy_name=None): + def del_policy(self, layer, component_name=None, policy_name=None): """ - Removes a policy. + Removes a policy. - :param dif: the dif to which the policy should be applied + :param layer: The layer. :param component_name: Name of the component. :param policy_name: Name of the policy. """ - self.policies[dif].del_policy(component_name, policy_name) + self.policies[layer].del_policy(component_name, policy_name) - def get_policy(self, dif): + def get_policy(self, layer): """ - :param dif: The DIF to get the policy of. + :param layer: The layer to get the policy of. :return: Returns the policy. """ - return self.policies[dif] + return self.policies[layer] def execute_commands(self, commands, as_root=False, time_out=3, use_proxy=False): """ - Execute a list of a commands on the node. + Execute a list of commands on the node. :param commands: A list of commands. :param as_root: Execute as root? :param time_out: Seconds before timing out. - :param use_proxy: Use a proxy to execute the commands? + :param use_proxy: Use a proxy? """ - return self.executor.execute_commands(self, - commands, - as_root, - time_out) + return self.executor.execute_commands( + self, commands, as_root, time_out) def execute_command(self, command, as_root=False, time_out=3, - use_proxy=False): + use_proxy=False, check=True): """ - Execute a single command on a node. + Execute a single command on the node. :param command: A command. :param as_root: Execute as root? :param time_out: Seconds before timing out. - :param use_proxy: Use a proxy to execute the commands? - :return: The stdout of the command. + :param use_proxy: Use a proxy? + :param check: If True, raise CommandError on non-zero exit. + :return: A CommandResult. """ - return self.executor.execute_command(self, - command, - as_root, - time_out) + return self.executor.execute_command( + self, command, as_root, time_out, check=check) def copy_file(self, path, destination): - """ - Copy file to node. - - :param path: Local location of the file. - :param destination: Destination location of the file. - """ + """Copy file to node.""" self.executor.copy_file(self, path, destination) def copy_files(self, paths, destination): - """ - Copy files to node. - - :param paths: Local location of the files. - :param destination: Destination location of the files. - """ + """Copy files to node.""" self.executor.copy_files(self, paths, destination) def fetch_file(self, path, destination, sudo=False): - """ - Fetch file from the node. - - :param path: Location of the files on the node. - :param destination: Destination location of the files. - :param sudo: The file is owned by root on the node? - """ + """Fetch file from node.""" self.executor.fetch_file(self, path, destination, sudo) def fetch_files(self, paths, destination, sudo=False): - """ - Fetch files from the node. - - :param paths: Location of the files on the node. - :param destination: Destination location of the files. - :param sudo: The file is owned by root on the node? - """ + """Fetch files from node.""" self.executor.fetch_files(self, paths, destination, sudo) - def set_link_state(self, dif, state): + def set_link_state(self, layer, state): """ Change the state of a link on the node. - :param dif: The name of the shim Ethernet DIF. + :param layer: The Ethernet layer. :param state: Up or down. """ - ipcp = self.get_ipcp_by_dif(dif) - self.execute_command('ip link set dev ' + ipcp.ifname + ' ' + state, - as_root=True) + ipcp = self.get_ipcp_by_layer(layer) + self.execute_command( + 'ip link set dev ' + ipcp.ifname + ' ' + state, + as_root=True) + +# --------------------------------------------------------------------------- +# IPCP classes +# --------------------------------------------------------------------------- class IPCP(object): - def __init__(self, name, node, dif): + def __init__(self, name, node, layer): self.name = name self.node = node - self.dif = dif + self.layer = layer self.registrations = [] - - # Is this IPCP the first in its DIF, so that it does not need - # to enroll to anyone ? - self.dif_bootstrapper = False + self.bootstrapper = False def __repr__(self): - return "{IPCP=%s,DIF=%s,N-1-DIFs=(%s)%s}" % \ - (self.name, self.dif.name, - ' '.join([dif.name for dif in self.registrations]), - ',bootstrapper' if self.dif_bootstrapper else '' - ) + return "{IPCP=%s,Layer=%s,N-1-Layers=(%s)%s}" % \ + (self.name, self.layer.name, + ' '.join([l.name for l in self.registrations]), + ',bootstrapper' if self.bootstrapper else '') def __hash__(self): - return hash((self.name, self.dif.name)) + return hash((self.name, self.layer.name)) def __eq__(self, other): return other is not None and self.name == other.name \ - and self.dif == other.dif + and self.layer == other.layer def __neq__(self, other): return not self == other -class ShimEthIPCP(IPCP): - def __init__(self, name, node, dif, ifname=None): - IPCP.__init__(self, name, node, dif) +class EthIPCP(IPCP): + def __init__(self, name, node, layer, ifname=None): + IPCP.__init__(self, name, node, layer) self.ifname = ifname -class ShimUDPIPCP(IPCP): - def __init__(self, name, node, dif): - IPCP.__init__(self, name, node, dif) - # TODO: add IP and port +class UdpIPCP(IPCP): + def __init__(self, name, node, layer, ip_addr=None, ifname=None): + IPCP.__init__(self, name, node, layer) + self.ip_addr = ip_addr or "0.0.0.0" + self.ifname = ifname + +# --------------------------------------------------------------------------- +# Policy +# --------------------------------------------------------------------------- class Policy(object): - def __init__(self, dif, node=None, policies=None): - self.dif = dif # type: NormalDIF + def __init__(self, layer, node=None, policies=None): + self.layer = layer self.node = node if policies is None: self._dict = dict() @@ -875,11 +918,9 @@ class Policy(object): self._dict = policies def add_policy(self, component_name, policy_name, **parameters): - self._dict.setdefault(component_name, dict())[policy_name] = parameters + self._dict.setdefault( + component_name, dict())[policy_name] = parameters - # - # Fetches effective policy info - # def get_policies(self, component_name=None, policy_name=None): policy = self._superimpose() if component_name is None: @@ -897,42 +938,38 @@ class Policy(object): else: del self._dict[component_name][policy_name] - # - # Merges this policy into that of its dif, obtaining - # the effective policy acting on self.node. - # def _superimpose(self): if self.node is None: return self - other = self.dif.policy + other = self.layer.policy base = dict(other._dict) base.update(self._dict) - return Policy(self.dif, self.node, base) + return Policy(self.layer, self.node, base) def __eq__(self, other): if not isinstance(other, Policy): return False - else: - return other.dif == self.dif \ - and other.node == self.node \ - and other._dict == self._dict + return (other.layer == self.layer + and other.node == self.node + and other._dict == self._dict) def __str__(self): node_str = (" Node: " + self.node) if self.node is not None else "" - return "Policy[Dif: %(dif)s,%(node_str)s Dict: %(dict)s]" \ - % {"dif": self.dif, "node_str": node_str, "dict": self._dict} + return "Policy[Layer: %(layer)s,%(node_str)s Dict: %(dict)s]" \ + % {"layer": self.layer, "node_str": node_str, + "dict": self._dict} def __repr__(self): node_str = (" Node: " + self.node) if self.node is not None else "" - s = "Policy[ Dif: %(dif)s,%(node_str)s" \ - % {"dif": self.dif, "node_str": node_str} + s = "Policy[ Layer: %(layer)s,%(node_str)s" \ + % {"layer": self.layer, "node_str": node_str} comps = [] for component in self._dict: for policy in self._dict[component]: - comps.append("\n Component %s has policy %s with params %s" - % (component, - policy, - self._dict[component][policy])) + comps.append( + "\n Component %s has policy %s with params %s" + % (component, policy, + self._dict[component][policy])) s += ",".join(comps) s += "\n]\n" return s diff --git a/rumba/executors/docker.py b/rumba/executors/docker.py index 8587878..4be9e9b 100644 --- a/rumba/executors/docker.py +++ b/rumba/executors/docker.py @@ -25,6 +25,7 @@ # from rumba import model as mod +from rumba.command import CommandResult import tempfile import tarfile @@ -44,16 +45,25 @@ class DockerExecutor(mod.Executor): self.testbed = testbed self.running_containers = testbed.running_containers - def execute_command(self, node, command, sudo=False, time_out=3): + def execute_command(self, node, command, sudo=False, time_out=3, + check=True): logger.debug("%s >> %s" % (node.name, command)) c, o = self.running_containers[node.name].exec_run(["sh", "-c", command]) - if c: - raise DockerException('A remote command returned an error. ' - 'Output:\n\n\t' + o.decode("utf-8")) - return o.decode("utf-8") + result = CommandResult( + exit_code=c, + stdout=o.decode("utf-8").rstrip(), + stderr="", + command=command, + node=node.name + ) + + if check: + result.check() + + return result def fetch_file(self, node, path, destination, as_root=False): if not path.startswith("/"): diff --git a/rumba/executors/local.py b/rumba/executors/local.py index cab626d..00a0adf 100644 --- a/rumba/executors/local.py +++ b/rumba/executors/local.py @@ -25,6 +25,7 @@ # from rumba import model as mod +from rumba.command import CommandResult import subprocess from shutil import copy @@ -38,17 +39,33 @@ class LocalExecutor(mod.Executor): def __init__(self, testbed): self.testbed = testbed - def execute_command(self, node, cmd, sudo=False, time_out=3): + def execute_command(self, node, cmd, sudo=False, time_out=3, + check=True): + if sudo: + cmd = "sudo " + cmd + logger.debug("%s >> %s" % (node.name, cmd)) try: - logger.debug("%s >> %s" % (node.name, cmd)) - output = subprocess.check_output(cmd, - universal_newlines=True, - shell=True) - return output.rstrip() - except subprocess.CalledProcessError as e: - logger.error("Return code was " + str(e.returncode)) + proc = subprocess.run(cmd, shell=True, + capture_output=True, + universal_newlines=True, + timeout=time_out) + except subprocess.TimeoutExpired: + logger.error("Command timed out after %ss: %s", time_out, cmd) raise + result = CommandResult( + exit_code=proc.returncode, + stdout=proc.stdout.rstrip(), + stderr=proc.stderr.rstrip(), + command=cmd, + node=node.name + ) + + if check: + result.check() + + return result + def fetch_file(self, node, path, destination, as_root=False): copy(path, destination) diff --git a/rumba/executors/ssh.py b/rumba/executors/ssh.py index a53b978..b3bc366 100644 --- a/rumba/executors/ssh.py +++ b/rumba/executors/ssh.py @@ -33,8 +33,13 @@ class SSHExecutor(Executor): def __init__(self, testbed): self.testbed = testbed - def execute_command(self, node, command, as_root=False, time_out=3): - return self.execute_commands(node, [command], as_root, time_out) + def execute_command(self, node, command, as_root=False, time_out=3, + check=True): + if as_root and node.ssh_config.username != 'root': + command = "sudo %s" % command + + return execute_command(self.testbed, node.ssh_config, command, + time_out, node_name=node.name, check=check) def execute_commands(self, node, commands, as_root=False, time_out=3): if as_root: @@ -42,7 +47,7 @@ class SSHExecutor(Executor): commands = list(map(lambda c: "sudo %s" % (c,), commands)) return execute_commands(self.testbed, node.ssh_config, commands, - time_out) + time_out, node_name=node.name) def fetch_file(self, node, path, destination, sudo=False): copy_file_from_testbed(self.testbed, node.ssh_config, path, diff --git a/rumba/irm_backend.py b/rumba/irm_backend.py new file mode 100644 index 0000000..e9cc5c6 --- /dev/null +++ b/rumba/irm_backend.py @@ -0,0 +1,343 @@ +# +# Rumba - IRM Backend Abstraction +# +# Copyright (C) 2017-2026 imec +# +# Dimitri Staessens <dimitri.staessens@ugent.be> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., http://www.fsf.org/about/contact/. +# + +""" +IRM backend abstraction for Ouroboros. + +Provides two implementations: +- IrmPython: uses pyouroboros CFFI bindings (in-process, local testbed) +- IrmCLI: generates irm CLI commands for SSH execution (remote testbeds) +""" + +import abc +import rumba.log as log +from rumba.elements.topology import LayerType + +logger = log.get_logger(__name__) + + +class IrmBackend(abc.ABC): + """ + Abstract interface for IRM operations. + + Maps to the pyouroboros irm API. Implementations execute the + operations either locally (via CFFI) or remotely (via CLI commands). + """ + + @abc.abstractmethod + def create_ipcp(self, node, name, layer_type): + """ + Create an IPCP process. + + :param node: The node to create the IPCP on. + :param name: Name for the IPCP. + :param layer_type: LayerType enum value. + :return: IPCP identifier (PID for local, name for remote). + """ + pass + + @abc.abstractmethod + def bootstrap_ipcp(self, node, name, layer_type, layer_name, + policies=None, eth_dev=None, ip_addr=None, + autobind=True): + """ + Bootstrap an IPCP into a layer. + + :param node: The node. + :param name: IPCP name. + :param layer_type: LayerType enum value. + :param layer_name: Name of the layer to bootstrap. + :param policies: Dict of component->policy mappings. + :param eth_dev: Ethernet device name (for eth-dix/eth-llc). + :param ip_addr: IP address (for UDP shim). + :param autobind: Automatically bind the IPCP name. + """ + pass + + @abc.abstractmethod + def enroll_ipcp(self, node, name, dst_name, autobind=True): + """ + Enroll an IPCP into an existing layer. + + :param node: The node. + :param name: IPCP name. + :param dst_name: Name of the destination IPCP to enroll with. + :param autobind: Automatically bind the IPCP name. + """ + pass + + @abc.abstractmethod + def connect_ipcp(self, node, name, dst_name): + """ + Create a data transfer connection between IPCPs. + + :param node: The node. + :param name: IPCP name. + :param dst_name: Destination IPCP name. + """ + pass + + @abc.abstractmethod + def reg_name(self, node, name, ipcp_names): + """ + Register a name in N-1 layers. + + :param node: The node. + :param name: The name to register. + :param ipcp_names: List of IPCP names to register with. + """ + pass + + @abc.abstractmethod + def bind_process(self, node, pid, name): + """ + Bind a process to a name. + + :param node: The node. + :param pid: Process ID. + :param name: Name to bind to. + """ + pass + + @abc.abstractmethod + def destroy_ipcp(self, node, name): + """ + Destroy an IPCP. + + :param node: The node. + :param name: IPCP name to destroy. + """ + pass + + +# ----------------------------------------------------------------------- +# Remote IRM: generates CLI commands for SSH execution +# ----------------------------------------------------------------------- + +# Map LayerType to Ouroboros irm CLI type strings +_LAYER_TYPE_TO_CLI = { + LayerType.LOCAL: 'local', + LayerType.UNICAST: 'unicast', + LayerType.BROADCAST: 'broadcast', + LayerType.ETH_LLC: 'eth-llc', + LayerType.ETH_DIX: 'eth-dix', + LayerType.UDP4: 'udp4', + LayerType.UDP6: 'udp6', +} + + +class IrmCLI(IrmBackend): + """ + IRM backend that generates ``irm`` CLI commands and executes them + on the node via its executor (SSH, Docker exec, etc.). + """ + + def create_ipcp(self, node, name, layer_type): + type_str = _LAYER_TYPE_TO_CLI[layer_type] + cmd = "irm i c n %s type %s" % (name, type_str) + node.execute_command(cmd, time_out=None) + return name # remote: we track by name + + def bootstrap_ipcp(self, node, name, layer_type, layer_name, + policies=None, eth_dev=None, ip_addr=None, + autobind=True): + type_str = _LAYER_TYPE_TO_CLI[layer_type] + cmd = "irm i b n %s type %s" % (name, type_str) + + if layer_type in (LayerType.ETH_DIX, LayerType.ETH_LLC): + if eth_dev: + cmd += " dev %s" % eth_dev + + if layer_type in (LayerType.UDP4, LayerType.UDP6): + if ip_addr: + cmd += " ip %s" % ip_addr + + if policies: + for comp, pol in policies.items(): + cmd += " %s %s" % (comp, pol) + + cmd += " layer %s" % layer_name + + if autobind: + cmd += " autobind" + + node.execute_command(cmd, time_out=None) + + def enroll_ipcp(self, node, name, dst_name, autobind=True): + cmd = "irm i e n %s dst %s" % (name, dst_name) + if autobind: + cmd += " autobind" + node.execute_command(cmd, time_out=None) + + def connect_ipcp(self, node, name, dst_name): + cmd = "irm i conn n %s dst %s" % (name, dst_name) + node.execute_command(cmd, time_out=None) + + def reg_name(self, node, name, ipcp_names): + cmd = "irm n r %s" % name + for ipcp_name in ipcp_names: + cmd += " ipcp %s" % ipcp_name + node.execute_command(cmd, time_out=None) + + def bind_process(self, node, pid, name): + cmd = "irm b process %s name %s" % (pid, name) + node.execute_command(cmd, time_out=None) + + def destroy_ipcp(self, node, name): + cmd = "irm i d n %s" % name + node.execute_command(cmd, time_out=None) + + +# ----------------------------------------------------------------------- +# Local IRM: uses pyouroboros CFFI bindings +# ----------------------------------------------------------------------- + +class IrmPython(IrmBackend): + """ + IRM backend that uses pyouroboros CFFI bindings for direct + in-process IRM calls. Only works on the local machine where + the Ouroboros IRMd is running. + """ + + def __init__(self): + try: + from ouroboros import irm as _irm + from ouroboros import cli as _cli + self._irm = _irm + self._cli = _cli + logger.info("pyouroboros IRM backend loaded successfully") + except ImportError: + raise ImportError( + "pyouroboros is required for Python backend. " + "Install it from the pyouroboros repository.") + + def _get_ipcp_type(self, layer_type): + """Convert LayerType to pyouroboros IpcpType.""" + type_map = { + LayerType.LOCAL: self._irm.IpcpType.LOCAL, + LayerType.UNICAST: self._irm.IpcpType.UNICAST, + LayerType.BROADCAST: self._irm.IpcpType.BROADCAST, + LayerType.ETH_LLC: self._irm.IpcpType.ETH_LLC, + LayerType.ETH_DIX: self._irm.IpcpType.ETH_DIX, + LayerType.UDP4: self._irm.IpcpType.UDP4, + LayerType.UDP6: self._irm.IpcpType.UDP6, + } + return type_map[layer_type] + + def create_ipcp(self, node, name, layer_type): + ipcp_type = self._get_ipcp_type(layer_type) + pid = self._cli.create_ipcp(name, ipcp_type) + logger.debug("Created IPCP %s (pid=%d, type=%s)", + name, pid, layer_type.value) + return pid + + def bootstrap_ipcp(self, node, name, layer_type, layer_name, + policies=None, eth_dev=None, ip_addr=None, + autobind=True): + pid = self._cli.pid_of(name) + ipcp_type = self._get_ipcp_type(layer_type) + irm = self._irm + + # Build config based on type + if layer_type == LayerType.UNICAST: + uc_kwargs = {} + dt_kwargs = {} + routing_kwargs = {} + ls_kwargs = {} + + # Parse policies dict into pyouroboros config + if policies: + for comp, pol in policies.items(): + if comp == 'rmt.pff': + pol_map = { + 'lfa': irm.LinkStatePolicy.LFA, + 'ecmp': irm.LinkStatePolicy.ECMP, + 'simple': irm.LinkStatePolicy.SIMPLE, + } + if pol in pol_map: + ls_kwargs['pol'] = pol_map[pol] + + if ls_kwargs: + routing_kwargs['ls'] = irm.LinkStateConfig(**ls_kwargs) + if routing_kwargs: + dt_kwargs['routing'] = irm.RoutingConfig(**routing_kwargs) + if dt_kwargs: + uc_kwargs['dt'] = irm.DtConfig(**dt_kwargs) + + conf = irm.IpcpConfig( + ipcp_type=ipcp_type, + layer_name=layer_name, + unicast=irm.UnicastConfig(**uc_kwargs)) + + elif layer_type in (LayerType.ETH_DIX, LayerType.ETH_LLC): + eth_conf = irm.EthConfig(dev=eth_dev or "") + conf = irm.IpcpConfig( + ipcp_type=ipcp_type, + layer_name=layer_name, + eth=eth_conf) + + elif layer_type == LayerType.UDP4: + udp4_kwargs = {} + if ip_addr: + udp4_kwargs['ip_addr'] = ip_addr + conf = irm.IpcpConfig( + ipcp_type=ipcp_type, + layer_name=layer_name, + udp4=irm.Udp4Config(**udp4_kwargs)) + + elif layer_type == LayerType.UDP6: + udp6_kwargs = {} + if ip_addr: + udp6_kwargs['ip_addr'] = ip_addr + conf = irm.IpcpConfig( + ipcp_type=ipcp_type, + layer_name=layer_name, + udp6=irm.Udp6Config(**udp6_kwargs)) + + else: + conf = irm.IpcpConfig( + ipcp_type=ipcp_type, + layer_name=layer_name) + + self._cli.bootstrap_ipcp(pid, conf, autobind=autobind) + + logger.debug("Bootstrapped IPCP %s in layer %s", name, layer_name) + + def enroll_ipcp(self, node, name, dst_name, autobind=True): + pid = self._cli.pid_of(name) + self._cli.enroll_ipcp(pid, dst_name, autobind=autobind) + logger.debug("Enrolled IPCP %s via %s", name, dst_name) + + def connect_ipcp(self, node, name, dst_name): + pid = self._cli.pid_of(name) + self._cli.connect_ipcp(pid, dst_name) + logger.debug("Connected IPCP %s to %s", name, dst_name) + + def reg_name(self, node, name, ipcp_names): + self._cli.reg_name(name, ipcps=ipcp_names) + + def bind_process(self, node, pid, name): + self._cli.bind_process(int(pid), name) + + def destroy_ipcp(self, node, name): + self._cli.destroy_ipcp(name) + logger.debug("Destroyed IPCP %s", name) diff --git a/rumba/log.py b/rumba/log.py index 17ad22e..0b847fc 100644 --- a/rumba/log.py +++ b/rumba/log.py @@ -24,9 +24,11 @@ # Foundation, Inc., http://www.fsf.org/about/contact/. # +import atexit import logging import logging.handlers -import multiprocessing +from logging.handlers import QueueHandler, QueueListener +import queue import sys import time @@ -41,133 +43,13 @@ CRITICAL = logging.CRITICAL loggers_set = set() +_setup_done = False -mq = multiprocessing.Queue() +mq = queue.Queue() logging_listener = None -try: - from logging.handlers import QueueHandler -except ImportError: - # We are in python2 code - class QueueHandler(logging.Handler): - """ - This handler sends events to a queue. Typically, it would be used - together with a multiprocessing Queue to centralise logging to file - in one process (in a multi-process application), so as to avoid file - write contention between processes. - - This code is new in Python 3.2, but this class can be copy pasted into - user code for use with earlier Python versions. - """ - - # Copy-pasted as per above docstring from logging - - def __init__(self, queue): - logging.Handler.__init__(self) - self.queue = queue - - def enqueue(self, record): - self.queue.put_nowait(record) - - def prepare(self, record): - self.format(record) - record.msg = record.message - record.args = None - record.exc_info = None - return record - - def emit(self, record): - try: - self.enqueue(self.prepare(record)) - except Exception: - self.handleError(record) - -try: - from logging.handlers import QueueListener -except ImportError: - # We are in python2 code - import threading - try: - import Queue - except ImportError: - # Make it pythonX with 3.0 <= X <3.2 - import queue as Queue - - class QueueListener(object): - """ - This class implements an internal threaded listener which watches for - LogRecords being added to a queue, removes them and passes them to a - list of handlers for processing. - """ - - # Also copy-pasted - _sentinel = None - - def __init__(self, queue, respect_handler_level=False, *handlers): - self.queue = queue - self.handlers = handlers - self._stop = threading.Event() - self._thread = None - self.respect_handler_level = respect_handler_level - - def dequeue(self, block): - return self.queue.get(block) - - def start(self): - self._thread = t = threading.Thread(target=self._monitor) - t.setDaemon(True) - t.start() - - def prepare(self , record): - return record - - def handle(self, record): - record = self.prepare(record) - for handler in self.handlers: - if not self.respect_handler_level: - process = True - else: - process = record.levelno >= handler.level - if process: - handler.handle(record) - - def _monitor(self): - q = self.queue - has_task_done = hasattr(q, 'task_done') - while not self._stop.isSet(): - try: - record = self.dequeue(True) - if record is self._sentinel: - break - self.handle(record) - if has_task_done: - q.task_done() - except Queue.Empty: - pass - # There might still be records in the queue. - while True: - try: - record = self.dequeue(False) - if record is self._sentinel: - break - self.handle(record) - if has_task_done: - q.task_done() - except Queue.Empty: - break - - def enqueue_sentinel(self): - self.queue.put_nowait(self._sentinel) - - def stop(self): - self._stop.set() - self.enqueue_sentinel() - self._thread.join() - self._thread = None - - class RumbaFormatter(logging.Formatter): """ The `logging.Formatter` subclass used by Rumba @@ -202,12 +84,18 @@ class RumbaFormatter(logging.Formatter): def setup(): """Configures the logging framework with default values.""" - global mq + global mq, _setup_done + if _setup_done: + return + # Attach QueueHandler to root logger so messages flow through the queue queue_handler = QueueHandler(mq) queue_handler.setLevel(logging.DEBUG) - logging.basicConfig(handlers=[queue_handler], level=logging.DEBUG) - logging.getLogger('').setLevel(logging.ERROR) + root_logger = logging.getLogger('') + root_logger.addHandler(queue_handler) + root_logger.setLevel(logging.DEBUG) + # Configure rumba loggers to show INFO and above logging.getLogger('rumba').setLevel(logging.INFO) + _setup_done = True # Used for the first call, in order to configure logging @@ -302,7 +190,8 @@ def flush_and_kill_logging(): this does not happen. """ global logging_listener - logging_listener.stop() + if logging_listener is not None: + logging_listener.stop() class LogOptions(object): @@ -315,6 +204,8 @@ class LogOptions(object): logging_listener = QueueListener(mq) self.log_to_console() logging_listener.start() + # Configure logging levels + setup() @staticmethod def _get_handlers(): @@ -366,3 +257,4 @@ class LogOptions(object): options = LogOptions() # module instance used for configuration +atexit.register(flush_and_kill_logging) diff --git a/rumba/model.py b/rumba/model.py index 7306138..19ed8ff 100644 --- a/rumba/model.py +++ b/rumba/model.py @@ -27,15 +27,27 @@ import os import stat +from rumba.command import ( + CommandResult, + CommandError, +) + +from rumba.process import Process + from rumba.elements.topology import ( - DIF, - ShimEthDIF, - ShimUDPDIF, - NormalDIF, + LayerType, + Layer, + EthDixLayer, + EthLlcLayer, + UnicastLayer, + BroadcastLayer, + LocalLayer, + Udp4Layer, + Udp6Layer, IPCP, - ShimEthIPCP, - ShimUDPIPCP, + EthIPCP, + UdpIPCP, Node, SSHConfig, @@ -43,7 +55,8 @@ from rumba.elements.topology import ( LinkQuality, Delay, Loss, - Distribution + Distribution, + Policy, ) from rumba.elements.experimentation import ( @@ -55,15 +68,24 @@ from rumba.elements.experimentation import ( __all__ = [ - # Topology - "DIF", - "ShimEthDIF", - "ShimUDPDIF", - "NormalDIF", + "CommandResult", + "CommandError", + + "Process", + + "LayerType", + "Layer", + "EthDixLayer", + "EthLlcLayer", + "UnicastLayer", + "BroadcastLayer", + "LocalLayer", + "Udp4Layer", + "Udp6Layer", "IPCP", - "ShimEthIPCP", - "ShimUDPIPCP", + "EthIPCP", + "UdpIPCP", "Node", "SSHConfig", @@ -72,15 +94,14 @@ __all__ = [ "Delay", "Loss", "Distribution", + "Policy", - # Experimentation "Experiment", "Testbed", "Executor", "tmp_dir", - # Other - "cache_dir" + "cache_dir", ] diff --git a/rumba/multiprocess.py b/rumba/multiprocess.py index 508c9d9..56d0f72 100644 --- a/rumba/multiprocess.py +++ b/rumba/multiprocess.py @@ -24,16 +24,12 @@ # Foundation, Inc., http://www.fsf.org/about/contact/. # -import multiprocessing.dummy as multiprocessing -import sys +import contextlib +import queue +import threading import rumba.log as log -if sys.version_info[0] >= 3: - import contextlib -else: - import contextlib2 as contextlib - logger = log.get_logger(__name__) @@ -63,7 +59,6 @@ def call_in_parallel(name_list, argument_list, executor_list): def job(executor, name, m_queue, argument): try: - # m_queue.cancel_join_thread() logger.debug('Starting process "%s".' % (name,)) executor(argument) @@ -75,7 +70,7 @@ def call_in_parallel(name_list, argument_list, executor_list): m_queue.put(e) logger.debug('About to start spawning processes.') - queue = multiprocessing.Queue() + result_queue = queue.Queue() with contextlib.ExitStack() as stack: # This is a composite context manager. # After exiting the 'with' block, the __exit__ method of all @@ -84,12 +79,12 @@ def call_in_parallel(name_list, argument_list, executor_list): for i, e in enumerate(executor_list): stack.enter_context(ProcessContextManager( target=job, - args=(e, name_list[i], queue, argument_list[i]) + args=(e, name_list[i], result_queue, argument_list[i]) )) msg_to_be_read += 1 results = [] for _ in range(len(executor_list)): - result = queue.get() # This blocks until results are available + result = result_queue.get() # This blocks until results are available msg_to_be_read -= 1 results.append(result) for result in results: @@ -105,7 +100,7 @@ class ProcessContextManager(object): args = () if kwargs is None: kwargs = {} - self.process = multiprocessing.Process( + self.process = threading.Thread( target=target, args=tuple(args), kwargs=kwargs @@ -118,14 +113,7 @@ class ProcessContextManager(object): def __exit__(self, exc_type, exc_val, exc_tb): if exc_tb is not None or exc_val is not None or exc_tb is not None: logger.error('Subprocess error: %s.' % (type(exc_val).__name__,)) - try: - self.process.terminate() - self.process.join() - except AttributeError: - # We are using multiprocessing.dummy, so no termination. - # We trust the threads will die with the application - # (since we are shutting down anyway) - pass + self.process.join() return False else: self.process.join() diff --git a/rumba/process.py b/rumba/process.py new file mode 100644 index 0000000..6f37433 --- /dev/null +++ b/rumba/process.py @@ -0,0 +1,118 @@ +# +# Rumba - Process lifecycle management +# +# Copyright (C) 2017-2026 imec +# +# Dimitri Staessens <dimitri.staessens@ugent.be> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., http://www.fsf.org/about/contact/. +# + +import time +import signal + +import rumba.log as log + +logger = log.get_logger(__name__) + + +class Process: + """Managed background process with lifecycle operations. + + Wraps either a local subprocess.Popen handle or a remote PID + tracked via node.execute_command(). + """ + + def __init__(self, node, pid, command, popen=None): + """ + :param node: The node the process runs on (or None for local). + :param pid: The process ID. + :param command: The command that was started. + :param popen: A subprocess.Popen handle (local processes only). + """ + self.node = node + self.pid = int(pid) if pid is not None else None + self.command = command + self._popen = popen + + def is_alive(self): + """Check if the process is still running. + + :returns: True if the process is alive. + """ + if self._popen is not None: + return self._popen.poll() is None + + if self.node is None or self.pid is None: + return False + + result = self.node.execute_command( + 'kill -0 %d' % self.pid, as_root=True, check=False) + return result.succeeded + + def wait(self, timeout=None): + """Wait for the process to exit. + + :param timeout: Maximum seconds to wait (None = forever). + :returns: The exit code, or None if still running after timeout. + """ + if self._popen is not None: + try: + return self._popen.wait(timeout=timeout) + except Exception: + return None + + # For remote processes, poll until dead + deadline = None if timeout is None else time.time() + timeout + while True: + if not self.is_alive(): + return 0 # Process is dead; no way to get real exit code + if deadline is not None and time.time() >= deadline: + return None + time.sleep(0.5) + + def kill(self, sig=signal.SIGTERM): + """Send a signal to the process. + + :param sig: Signal number (default SIGTERM). + """ + if self._popen is not None: + self._popen.send_signal(sig) + return + + if self.node is None or self.pid is None: + return + + self.node.execute_command( + 'kill -%d %d' % (sig, self.pid), as_root=True, check=False) + + def output(self): + """Read stdout from the process (local Popen only). + + :returns: stdout string, or None if not available. + """ + if self._popen is not None: + try: + out, _ = self._popen.communicate(timeout=0) + return out + except Exception: + return None + return None + + def __repr__(self): + loc = "local" if self._popen else ( + self.node.name if self.node else "?") + return "Process(pid=%s, cmd='%s', on=%s)" % ( + self.pid, self.command, loc) diff --git a/rumba/prototypes/enroll.py b/rumba/prototypes/enroll.py deleted file mode 100755 index d3501cc..0000000 --- a/rumba/prototypes/enroll.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/env python -# -# A library to manage ARCFIRE experiments -# -# Copyright (C) 2017-2018 Nextworks S.r.l. -# Copyright (C) 2017-2018 imec -# -# Sander Vrijders <sander.vrijders@ugent.be> -# Dimitri Staessens <dimitri.staessens@ugent.be> -# Vincenzo Maffione <v.maffione@nextworks.it> -# Marco Capitani <m.capitani@nextworks.it> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., http://www.fsf.org/about/contact/. -# - -import argparse -import socket -import time -import re -import sys - -def printalo(byt): - print(repr(byt).replace('\\n', '\n')) - - -def get_response(s): - data = bytes() - while 1: - data += s.recv(1024) - lines = str(data).replace('\\n', '\n').split('\n') - #print(lines) - if lines[-1].find("IPCM") != -1: - return lines[:len(lines)-1] - - -description = "Python script to enroll IPCPs" -epilog = "2016 Vincenzo Maffione <v.maffione@nextworks.it>" - -argparser = argparse.ArgumentParser(description = description, - epilog = epilog) -argparser.add_argument('--ipcm-conf', help = "Path to the IPCM configuration file", - type = str, required = True) -argparser.add_argument('--enrollee-name', help = "Name of the enrolling IPCP", - type = str, required = True) -argparser.add_argument('--dif', help = "Name of DIF to enroll to", - type = str, required = True) -argparser.add_argument('--lower-dif', help = "Name of the lower level DIF", - type = str, required = True) -argparser.add_argument('--enroller-name', help = "Name of the remote neighbor IPCP to enroll to", - type = str, required = True) -args = argparser.parse_args() - -socket_name = None - -fin = open(args.ipcm_conf, 'r') -while 1: - line = fin.readline() - if line == '': - break - - m = re.search(r'"(\S+ipcm-console.sock)', line) - if m != None: - socket_name = m.group(1) - break -fin.close() - -if socket_name == None: - print('Cannot find %s' % (socket_name)) - quit(1) - -s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - -connected = False -trials = 0 -while trials < 4: - try: - s.connect(socket_name) - connected = True - break - except: - pass - trials += 1 - time.sleep(1) - -if connected: - try: - # Receive the banner - get_response(s) - - # Send the IPCP list command - cmd = u'list-ipcps\n' - s.sendall(cmd.encode('ascii')) - - # Get the list of IPCPs and parse it to look for the enroller ID - print('Looking up identifier for IPCP %s' % args.enrollee_name) - lines = get_response(s) - print(lines) - enrollee_id = None - for line in lines: - rs = r'^\s*(\d+)\s*\|\s*' + args.enrollee_name.replace('.', '\\.') - m = re.match(rs, line) - if m != None: - enrollee_id = m.group(1) - - if enrollee_id == None: - print('Could not find the ID of enrollee IPCP %s' \ - % args.enrollee_name) - raise Exception() - - # Send the enroll command - cmd = u'enroll-to-dif %s %s %s %s 1\n' \ - % (enrollee_id, args.dif, args.lower_dif, args.enroller_name) - print(cmd) - - s.sendall(cmd.encode('ascii')) - - # Get the enroll command answer - lines = get_response(s) - print(lines) - except: - s.close() - raise - -else: - print('Failed to connect to "%s"' % socket_name) - sys.exit(-1) - -s.close() diff --git a/rumba/prototypes/irati.py b/rumba/prototypes/irati.py deleted file mode 100644 index 3860d14..0000000 --- a/rumba/prototypes/irati.py +++ /dev/null @@ -1,467 +0,0 @@ -# -# A library to manage ARCFIRE experiments -# -# Copyright (C) 2017-2018 Nextworks S.r.l. -# Copyright (C) 2017-2018 imec -# -# Sander Vrijders <sander.vrijders@ugent.be> -# Dimitri Staessens <dimitri.staessens@ugent.be> -# Vincenzo Maffione <v.maffione@nextworks.it> -# Marco Capitani <m.capitani@nextworks.it> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., http://www.fsf.org/about/contact/. -# - -import copy -import json -import os -import time - -import rumba.ssh_support as ssh -import rumba.model as mod -import rumba.multiprocess as m_processing -import rumba.prototypes.irati_templates as irati_templates -import rumba.log as log - -logger = log.get_logger(__name__) - - -class Experiment(mod.Experiment): - """ - Represents an IRATI experiment. - """ - - @staticmethod - def make_executor(node, packages, testbed): - def executor(commands): - ssh.aptitude_install(testbed, node, packages) - node.execute_commands(commands, time_out=None, use_proxy=True) - return executor - - def prototype_name(self): - return 'irati' - - @staticmethod - def real_sudo(s): - return 'sudo ' + s - - @staticmethod - def fake_sudo(s): - return s - - def __init__(self, testbed, - nodes=None, - git_repo='https://github.com/IRATI/stack', - git_branch='arcfire', - installpath=None, - varpath=None, - app_mappings=None, - enrollment_strategy='minimal'): - """ - Initializes the experiment class. - - :param testbed: The testbed to run the experiment on. - :param nodes: The list of nodes. - :param git_repo: The git repository to use for installation. - :param git_branch: The branch of the git repository to use. - :param installpath: The installation path of IRATI. - :param varpath: The /var path of IRATI. - :param app_mappings: a list of application -> dif mapping containing - all application which will register to - any given dif. - :type app_mappings: `List[(str, str)]` - :param enrollment_strategy: Can be 'full-mesh', 'minimal' or 'manual'. - """ - mod.Experiment.__init__(self, - testbed, - nodes, - git_repo, - git_branch, - prototype_logs=['/tmp/ipcmstart.log'], - enrollment_strategy=enrollment_strategy) - if installpath is None: - installpath = '/usr' - if varpath is None: - varpath = '' - irati_templates.env_dict['installpath'] = installpath - irati_templates.env_dict['varpath'] = varpath - self.manager = False - self.conf_files = None - self.shim2vlan = {} - - if app_mappings is None: - app_mappings = [] - self.app_mappings = app_mappings - - if self.testbed.username == 'root': - self.sudo = self.fake_sudo - else: - self.sudo = self.real_sudo - - self._conf_dir = os.path.join(mod.tmp_dir, 'IRATI_conf') - try: - os.mkdir(self._conf_dir) - except OSError: - # Already there, nothing to do - pass - - def conf_dir(self, path): - return os.path.join(self._conf_dir, path) - - def install(self): - - packages = ["g++", "gcc", "libtool", "linux-headers-$(uname -r)", - "autoconf", "automake", "protobuf-compiler", - "libprotobuf-dev", "git", "pkg-config", "libssl-dev"] - - cmds = [self.sudo("rm -rf ~/stack"), - "cd ~; git clone -b " + self.git_branch + " " + self.git_repo, - "cd ~/stack && " - + self.sudo("./configure && ") + self.sudo("make install")] - names = [] - executors = [] - args = [] - for node in self.nodes: - - executor = self.make_executor(node, packages, self.testbed) - - names.append(node.name) - executors.append(executor) - args.append(cmds) - m_processing.call_in_parallel(names, args, executors) - - def bootstrap_network(self): - for node in self.nodes: - self.process_node(node) - self.enroll_nodes() - - def _install_prototype(self): - logger.info("installing IRATI on all the nodes") - self.install() - logger.info("installation complete") - - def _bootstrap_prototype(self): - logger.info("setting up") - self.conf_files = self.write_conf() - logger.info("configuration files generated for all nodes") - self.bootstrap_network() - logger.info("IPCPs created and enrolled on all nodes") - - def process_node(self, node): - name = node.name - - vlans = [] - for ipcp in node.ipcps: - if isinstance(ipcp, mod.ShimEthIPCP): - vlans.append([ipcp.ifname, self.shim2vlan[ipcp.dif.name]]) - - if vlans: - ssh.setup_vlans(self.testbed, node, vlans) - - gen_files_conf = self.conf_files[node] + ['da.map'] - dir_path = os.path.dirname(os.path.abspath(__file__)) - gen_files_bin = 'enroll.py' - gen_files_conf_full = [self.conf_dir(x) for x in gen_files_conf] - gen_files_bin_full = [os.path.join(dir_path, 'enroll.py')] - - ipcm_components = ['scripting', 'console'] - if self.manager: - ipcm_components.append('mad') - ipcm_components = ', '.join(ipcm_components) - - gen_files = gen_files_conf_full + gen_files_bin_full - - format_args = {'name': name, - 'ssh': node.ssh_config.port, - 'username': self.testbed.username, - 'genfiles': gen_files, - 'genfilesconf': ' '.join(gen_files_conf), - 'genfilesbin': gen_files_bin, - 'verb': 'DBG', - 'ipcmcomps': ipcm_components} - - logger.info('Copying configuration files to node %s', node.name) - ssh.copy_files_to_testbed(self.testbed, - node.ssh_config, - gen_files, - '') - - cmds = [self.sudo('hostname %(name)s' % format_args), - self.sudo('modprobe rina-irati-core'), - self.sudo('chmod a+rw /dev/irati'), - # The || true should be removed soon, but it's needed - # until we update the bitbucket repo. - self.sudo('chmod a+rw /dev/irati-ctrl || true'), - self.sudo('mv %(genfilesconf)s /etc' % format_args), - self.sudo('mv %(genfilesbin)s /usr/bin') % format_args, - self.sudo('chmod a+x /usr/bin/enroll.py') % format_args] - - cmds += [self.sudo('modprobe rina-default-plugin'), - self.sudo('modprobe shim-eth-vlan'), - self.sudo('modprobe normal-ipcp'), - self.sudo('ipcm -a \"%(ipcmcomps)s\" ' - '-c /etc/%(name)s.ipcm.conf -l %(verb)s ' - '> /tmp/ipcmstart.log 2>&1 &' - % format_args)] - - logger.info('Sending setup commands to node %s.', node.name) - ssh.execute_commands(self.testbed, node.ssh_config, cmds) - - def enroll_nodes(self): - logger.info("Starting enrollment phase.") - time.sleep(5) - for enrollment_list in self.enrollments: - for e in enrollment_list: - logger.info( - 'Enrolling %s to DIF %s against neighbor %s,' - ' through lower DIF %s.', - e['enrollee'].name, - e['dif'].name, - e['enroller'].name, - e['lower_dif'].name) - - time.sleep(1) # Important! - - e_args = {'ldif': self.dif_name(e['lower_dif']), - 'dif': e['dif'].name, - 'nname': e['enrollee'].node.name, - 'iname': e['enrollee'].name, - 'o_iname': e['enroller'].name} - - cmd = self.sudo('enroll.py --lower-dif %(ldif)s --dif %(dif)s ' - '--ipcm-conf /etc/%(nname)s.ipcm.conf ' - '--enrollee-name %(iname)s.IPCP ' - '--enroller-name %(o_iname)s.IPCP' - % e_args) - ssh.execute_command(self.testbed, - e['enrollee'].node.ssh_config, - cmd) - - def dif_name(self, dif): - try: - return str(self.shim2vlan[dif.name]) - except KeyError: - return dif.name - - def write_conf(self): - # Constants and initializations - ipcmconfs = dict() - difconfs = dict() - ipcp2shim_map = {} - node2id_map = {} - mgmt_dif_name = 'NMS' - conf_files = {} # dict of per-nod conf files - - # Translating Shim Eth difs to vlan tags. - next_vlan = 10 - for dif in self.dif_ordering: - if isinstance(dif, mod.ShimEthDIF): - try: - vlan = int(dif.name) - self.shim2vlan[dif.name] = vlan - except ValueError: - vlan = next_vlan - next_vlan += 10 - self.shim2vlan[dif.name] = vlan - - # If some app directives were specified, use those to build da.map. - # Otherwise, assume the standard applications are to be mapped in - # the DIF with the highest rank. - app_mappings = self.app_mappings - if len(app_mappings) == 0: - if len(self.dif_ordering) > 0: - for adm in \ - irati_templates.da_map_base["applicationToDIFMappings"]: - adm["difName"] = "%s" % (self.dif_ordering[-1].name,) - else: # not yet supported - irati_templates.da_map_base["applicationToDIFMappings"] = [] - for app, dif in app_mappings: - irati_templates.da_map_base["applicationToDIFMappings"]\ - .append({"encodedAppName": app, - "difName": dif - }) - - if self.manager: - # Add MAD/Manager configuration - irati_templates.get_ipcmconf_base()["addons"] = { - "mad": { - "managerAppName": "", - "NMSDIFs": [{"DIF": "%s" % mgmt_dif_name}], - "managerConnections": [{ - "managerAppName": "manager-1--", - "DIF": "%s" % mgmt_dif_name - }] - } - } - - node_number = 1 - for node in self.nodes: # type: mod.Node - node2id_map[node.name] = node_number - node_number += 1 - ipcmconfs[node.name] = copy.deepcopy(irati_templates.get_ipcmconf_base()) - if self.manager: - ipcmconfs[node.name]["addons"]["mad"]["managerAppName"] \ - = "%s.mad-1--" % (node.name,) - - for dif in self.dif_ordering: # type: mod.DIF - if isinstance(dif, mod.ShimEthDIF): - ipcp2shim_map.update({ipcp.name: dif for ipcp in dif.ipcps}) - elif isinstance(dif, mod.NormalDIF): - difconfs[dif.name] = dict() - # Generate base conf - dif_conf = copy.deepcopy(irati_templates.normal_dif_base) - # push qos_cubes - if len(dif.qos_cubes) != 0: - dif_conf["qosCubes"] = [] - for cube in dif.qos_cubes: - dif_conf["qosCubes"].append( - irati_templates.generate_qos_cube(**cube) - ) - if dif.add_default_qos_cubes: - # Add basic cubes - unreliable = copy.deepcopy(irati_templates.qos_cube_u_base) - unreliable["id"] = len(dif_conf["qosCubes"]) + 1 - dif_conf["qosCubes"].append(unreliable) - reliable = copy.deepcopy(irati_templates.qos_cube_r_base) - reliable["id"] = len(dif_conf["qosCubes"]) + 1 - dif_conf["qosCubes"].append(reliable) - - for node in dif.members: - difconfs[dif.name][node.name] = copy.deepcopy( - dif_conf - ) - - for node in self.nodes: # type: mod.Node - ipcmconf = ipcmconfs[node.name] - - for ipcp in node.ipcps: # type: mod.ShimEthIPCP - if isinstance(ipcp, mod.ShimEthIPCP): - shim = ipcp2shim_map[ipcp.name] # type: mod.ShimEthDIF - shim_name = self.dif_name(shim) - ipcmconf["ipcProcessesToCreate"].append({ - "apName": "eth.%s.IPCP" % ipcp.name, - "apInstance": "1", - "difName": shim_name - }) - - template_file_name = self.conf_dir( - 'shimeth.%s.%s.dif' - % (node.name, shim_name)) - ipcmconf["difConfigurations"].append({ - "name": shim_name, - "template": os.path.basename(template_file_name) - }) - - fout = open(template_file_name, 'w') - fout.write(json.dumps( - {"difType": "shim-eth-vlan", - "configParameters": { - "interface-name": ipcp.ifname - } - }, - indent=4, sort_keys=True)) - fout.close() - conf_files.setdefault(node, []).append( - 'shimeth.%s.%s.dif' - % (node.name, shim_name)) - - # Run over dif_ordering array, to make sure each IPCM config has - # the correct ordering for the ipcProcessesToCreate list of operations. - # If we iterated over the difs map, the order would be randomic, and so - # some IPCP registrations in lower DIFs may fail. - # This would happen because at the moment of registration, - # it may be that the IPCP of the lower DIF has not been created yet. - shims = ipcp2shim_map.values() - for dif in self.dif_ordering: # type: mod.NormalDIF - - if dif in shims: - # Shims are managed separately, in the previous loop - continue - - for node in dif.members: # type: mod.Node - node_name = node.name - ipcmconf = ipcmconfs[node_name] - - normal_ipcp = {"apName": "%s.%s.IPCP" % (dif.name, node_name), - "apInstance": "1", - "difName": "%s" % (dif.name,), - "difsToRegisterAt": []} - - for lower_dif in node.dif_registrations[dif]: # type: mod.DIF - normal_ipcp["difsToRegisterAt"].append( - self.dif_name(lower_dif)) - - ipcmconf["ipcProcessesToCreate"].append(normal_ipcp) - - ipcmconf["difConfigurations"].append({ - "name": "%s" % (dif.name,), - "template": "normal.%s.%s.dif" % (node_name, dif.name,) - }) - - # Fill in the map of IPCP addresses. - # This could be moved at difconfs - for other_node in dif.members: # type: mod.Node - difconfs[dif.name][other_node.name] \ - ["knownIPCProcessAddresses"].append({ - "apName": "%s.%s.IPCP" % (dif.name, node_name), - "apInstance": "1", - "address": 16 + node2id_map[node_name]}) - policy_dict = node.get_policy(dif).get_policies() - for component in policy_dict: - for policy_name in policy_dict[component]: - params = policy_dict[component][policy_name].items() - irati_templates.translate_policy( - difconfs[dif.name][node_name], - component, - policy_name, - params - ) - - # Dump the DIF Allocator map - with open(self.conf_dir('da.map'), 'w') as da_map_file: - json.dump(irati_templates.da_map_base, - da_map_file, - indent=4, - sort_keys=True) - - for node in self.nodes: - # Dump the IPCM configuration files - with open(self.conf_dir('%s.ipcm.conf' - % (node.name,)), 'w') as node_file: - json.dump(ipcmconfs[node.name], - node_file, - indent=4, - sort_keys=True) - conf_files.setdefault(node, []).append( - '%s.ipcm.conf' % (node.name,)) - - for dif in self.dif_ordering: # type: mod.DIF - dif_conf = difconfs.get(dif.name, None) - if dif_conf: - # Dump the normal DIF configuration files - for node in dif.members: - with open(self.conf_dir('normal.%s.%s.dif' - % (node.name, dif.name)), 'w') \ - as dif_conf_file: - json.dump(dif_conf[node.name], - dif_conf_file, - indent=4, - sort_keys=True) - conf_files.setdefault(node, []).append( - 'normal.%s.%s.dif' % (node.name, dif.name)) - return conf_files - - def _terminate_prototype(self): - return diff --git a/rumba/prototypes/irati_templates.py b/rumba/prototypes/irati_templates.py deleted file mode 100644 index a6dd3c4..0000000 --- a/rumba/prototypes/irati_templates.py +++ /dev/null @@ -1,437 +0,0 @@ -# -# A library to manage ARCFIRE experiments -# -# Copyright (C) 2017-2018 Nextworks S.r.l. -# Copyright (C) 2017-2018 imec -# -# Sander Vrijders <sander.vrijders@ugent.be> -# Dimitri Staessens <dimitri.staessens@ugent.be> -# Vincenzo Maffione <v.maffione@nextworks.it> -# Marco Capitani <m.capitani@nextworks.it> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., http://www.fsf.org/about/contact/. -# - -# Environment setup for VMs. Standard linux approach -env_dict = {'installpath': '/usr', 'varpath': ''} - - -# Template for a IPCM configuration file -def get_ipcmconf_base(): - return { - "configFileVersion": "1.4.1", - "localConfiguration": { - "installationPath": "%(installpath)s/bin" % env_dict, - "libraryPath": "%(installpath)s/lib" % env_dict, - "logPath": "%(varpath)s/var/log" % env_dict, - "consoleSocket": "%(varpath)s/var/run/ipcm-console.sock" % env_dict, - "pluginsPaths": [ - "%(installpath)s/lib/rinad/ipcp" % env_dict, - "/lib/modules/4.9.28-irati/extra" - ] - }, - - "ipcProcessesToCreate": [], - "difConfigurations": [], -} - - -da_map_base = { - "applicationToDIFMappings": [ - { - "encodedAppName": "rina.apps.echotime.server-1--", - "difName": "n.DIF" - }, - { - "encodedAppName": "traffic.generator.server-1--", - "difName": "n.DIF" - } - ], -} - - -def generate_qos_cube( - name, - cube_id, - initial_credit=200, - ordered=False, - delay=None, - loss=None, - reliable=False, - data_rxms_nmax=5, - initial_rtx_time=1000 -): - cube = { - "name": name, - "id": cube_id, - "partialDelivery": False, - "orderedDelivery": ordered, - "efcpPolicies": { - "dtpPolicySet": { - "name": "default", - "version": "0" - }, - "initialATimer": 0, - "dtcpPresent": True, - "dtcpConfiguration": { - "dtcpPolicySet": { - "name": "default", - "version": "0" - }, - "rtxControl": False, - "flowControl": True, - "flowControlConfig": { - "rateBased": False, - "windowBased": True, - "windowBasedConfig": { - "maxClosedWindowQueueLength": 10, - "initialCredit": initial_credit - } - } - } - } - } - if delay is not None: - cube["delay"] = delay - if loss is not None: - cube["loss"] = loss - if reliable: - cube["maxAllowableGap"] = 0 - cube["efcpPolicies"]["dtcpConfiguration"]["rtxControl"] = True - cube["efcpPolicies"]["dtcpConfiguration"]["rtxControlConfig"] = { - "dataRxmsNmax": data_rxms_nmax, - "initialRtxTime": initial_rtx_time - } - return cube - - -qos_cube_u_base = { - "name": "unreliablewithflowcontrol", - "id": 1, - "partialDelivery": False, - "orderedDelivery": True, - "efcpPolicies": { - "dtpPolicySet": { - "name": "default", - "version": "0" - }, - "initialATimer": 0, - "dtcpPresent": True, - "dtcpConfiguration": { - "dtcpPolicySet": { - "name": "default", - "version": "0" - }, - "rtxControl": False, - "flowControl": True, - "flowControlConfig": { - "rateBased": False, - "windowBased": True, - "windowBasedConfig": { - "maxClosedWindowQueueLength": 10, - "initialCredit": 200 - } - } - } - } -} - -qos_cube_r_base = { - "name": "reliablewithflowcontrol", - "id": 2, - "partialDelivery": False, - "orderedDelivery": True, - "maxAllowableGap": 0, - "efcpPolicies": { - "dtpPolicySet": { - "name": "default", - "version": "0" - }, - "initialATimer": 0, - "dtcpPresent": True, - "dtcpConfiguration": { - "dtcpPolicySet": { - "name": "default", - "version": "0" - }, - "rtxControl": True, - "rtxControlConfig": { - "dataRxmsNmax": 5, - "initialRtxTime": 1000 - }, - "flowControl": True, - "flowControlConfig": { - "rateBased": False, - "windowBased": True, - "windowBasedConfig": { - "maxClosedWindowQueueLength": 10, - "initialCredit": 200 - } - } - } - } -} - - -# Template for a normal DIF configuration file -normal_dif_base = { - "difType": "normal-ipc", - "dataTransferConstants": { - "addressLength": 2, - "cepIdLength": 2, - "lengthLength": 2, - "portIdLength": 2, - "qosIdLength": 2, - "rateLength": 4, - "frameLength": 4, - "sequenceNumberLength": 4, - "ctrlSequenceNumberLength": 4, - "maxPduSize": 10000, - "maxPduLifetime": 60000 - }, - - "qosCubes": [ - qos_cube_u_base, qos_cube_r_base - ], - - "knownIPCProcessAddresses": [], - - "addressPrefixes": [ - { - "addressPrefix": 0, - "organization": "N.Bourbaki" - }, { - "addressPrefix": 16, - "organization": "IRATI" - } - ], - - "rmtConfiguration": { - "pffConfiguration": { - "policySet": { - "name": "default", - "version": "0" - } - }, - "policySet": { - "name": "default", - "version": "1" - } - }, - - "enrollmentTaskConfiguration": { - "policySet": { - "name": "default", - "version": "1", - "parameters": [ - { - "name": "enrollTimeoutInMs", - "value": "10000" - }, { - "name": "watchdogPeriodInMs", - "value": "30000" - }, { - "name": "declaredDeadIntervalInMs", - "value": "120000" - }, { - "name": "neighborsEnrollerPeriodInMs", - "value": "0" - }, { - "name": "maxEnrollmentRetries", - "value": "0" - } - ] - } - }, - - "flowAllocatorConfiguration": { - "policySet": { - "name": "default", - "version": "1" - } - }, - - "namespaceManagerConfiguration": { - "policySet": { - "name": "default", - "version": "1" - } - }, - - "securityManagerConfiguration": { - "policySet": { - "name": "default", - "version": "1" - } - }, - - "resourceAllocatorConfiguration": { - "pduftgConfiguration": { - "policySet": { - "name": "default", - "version": "0" - } - } - }, - - "routingConfiguration": { - "policySet": { - "name": "link-state", - "version": "1", - "parameters": [ - { - "name": "objectMaximumAge", - "value": "10000" - }, { - "name": "waitUntilReadCDAP", - "value": "5001" - }, { - "name": "waitUntilError", - "value": "5001" - }, { - "name": "waitUntilPDUFTComputation", - "value": "103" - }, { - "name": "waitUntilFSODBPropagation", - "value": "101" - }, { - "name": "waitUntilAgeIncrement", - "value": "997" - }, { - "name": "routingAlgorithm", - "value": "Dijkstra" - } - ] - } - } -} - - -def ps_set(d, k, v, parms): - if k not in d: - d[k] = {'name': '', 'version': '1'} - - if d[k]["name"] == v and "parameters" in d[k]: - cur_names = [p["name"] for p in d[k]["parameters"]] - for p in parms: - name, value = p - if name in cur_names: - for i in range(len(d[k]["parameters"])): - if d[k]["parameters"][i]["name"] == name: - d[k]["parameters"][i]["value"] = value - break - else: - d[k]["parameters"].append({'name': name, 'value': value}) - - elif len(parms) > 0: - d[k]["parameters"] = [ - {'name': p[0], 'value': p[1]} - for p in parms] - - d[k]["name"] = v - - -def dtp_ps_set(d, v, parms): - for i in range(len(d["qosCubes"])): - ps_set(d["qosCubes"][i]["efcpPolicies"], "dtpPolicySet", v, parms) - - -def dtcp_ps_set(d, v, parms): - for i in range(len(d["qosCubes"])): - ps_set(d["qosCubes"][i]["efcpPolicies"]["dtcpConfiguration"], - "dtcpPolicySet", v, parms) - - -policy_translator = { - 'rmt.pff': lambda d, v, p: ps_set(d["rmtConfiguration"]["pffConfiguration"], - "policySet", v, p), - 'rmt': lambda d, v, p: ps_set(d["rmtConfiguration"], "policySet", v, p), - 'enrollment-task': lambda d, v, p: ps_set(d["enrollmentTaskConfiguration"], - "policySet", v, p), - 'flow-allocator': lambda d, v, p: ps_set(d["flowAllocatorConfiguration"], - "policySet", v, p), - 'namespace-manager': lambda d, v, p: ps_set( - d["namespaceManagerConfiguration"], "policySet", v, p), - 'security-manager': lambda d, v, p: ps_set( - d["securityManagerConfiguration"], "policySet", v, p), - 'routing': lambda d, v, p: ps_set( - d["routingConfiguration"], "policySet", v, p), - 'resource-allocator.pduftg': lambda d, v, p: ps_set( - d["resourceAllocatorConfiguration"]["pduftgConfiguration"], "policySet", v, p), - 'efcp.*.dtcp': None, - 'efcp.*.dtp': None, -} - - -def is_security_path(path): - sp = path.split('.') - return (len(sp) == 3) and (sp[0] == 'security-manager') \ - and (sp[1] in ['auth', 'encrypt', 'ttl', 'errorcheck']) - - -# Do we know this path ? -def policy_path_valid(path): - if path in policy_translator: - return True - - # Try to validate security configuration - if is_security_path(path): - return True - - return False - - -def translate_security_path(d, path, ps, parms): - u1, component, profile = path.split('.') - if "authSDUProtProfiles" not in d["securityManagerConfiguration"]: - d["securityManagerConfiguration"]["authSDUProtProfiles"] = {} - d = d["securityManagerConfiguration"]["authSDUProtProfiles"] - - tr = {'auth': 'authPolicy', 'encrypt': 'encryptPolicy', - 'ttl': 'TTLPolicy', 'errorcheck': 'ErrorCheckPolicy'} - - if profile == 'default': - if profile not in d: - d["default"] = {} - - ps_set(d["default"], tr[component], ps, parms) - - else: # profile is the name of a DIF - if "specific" not in d: - d["specific"] = [] - j = -1 - for i in range(len(d["specific"])): - if d["specific"][i]["underlyingDIF"] == profile + ".DIF": - j = i - break - - if j == -1: # We need to create an entry for the new DIF - d["specific"].append({"underlyingDIF": profile + ".DIF"}) - - ps_set(d["specific"][j], tr[component], ps, parms) - - -def translate_policy(difconf, path, ps, parms): - if path == 'efcp.*.dtcp': - dtcp_ps_set(difconf, ps, parms) - - elif path == 'efcp.*.dtp': - dtp_ps_set(difconf, ps, parms) - - elif is_security_path(path): - translate_security_path(difconf, path, ps, parms) - - else: - policy_translator[path](difconf, ps, parms) diff --git a/rumba/prototypes/ouroboros.py b/rumba/prototypes/ouroboros.py index 9cdd3c9..ce05181 100644 --- a/rumba/prototypes/ouroboros.py +++ b/rumba/prototypes/ouroboros.py @@ -33,8 +33,12 @@ import rumba.model as mod import rumba.multiprocess as m_processing import rumba.log as log import rumba.testbeds.local as local +import rumba.testbeds.localnet as localnet import rumba.testbeds.dockertb as docker import rumba.storyboard as sb +from rumba.elements.topology import LayerType +from rumba.irm_backend import IrmPython, IrmCLI +from rumba.process import Process logger = log.get_logger(__name__) @@ -53,7 +57,7 @@ class OurServer(sb.Server): server.min_duration, server.id, server.as_root, - server.difs + server.layers ) def _make_run_cmd(self, node): @@ -64,12 +68,12 @@ class OurServer(sb.Server): # Build register command r_cmd = 'irm n r %s ' % (self.id,) - if len(self.difs) == 0: + if len(self.layers) == 0: r_cmd += ' '.join('ipcp %s' % (ipcp.name,) for ipcp in node.ipcps) else: - for dif in self.difs: + for layer in self.layers: for ipcp in node.ipcps: - if ipcp.dif is dif: + if ipcp.layer is layer: r_cmd += 'ipcp %s' % (ipcp.name,) r_cmd += ' && ' @@ -143,6 +147,20 @@ class Experiment(mod.Experiment): self.set_startup_command("irmd") self.metrics_python_version = "python3.9" + # Create IRM backend + if isinstance(testbed, (local.Testbed, + localnet.LocalNetworkTestbed)): + try: + self.irm = IrmPython() + logger.info("Using pyouroboros IRM backend") + except ImportError: + self.irm = IrmCLI() + logger.info("pyouroboros not available, " + "using CLI IRM backend for local testbed") + else: + self.irm = IrmCLI() + logger.info("Using CLI IRM backend for remote testbed") + @staticmethod def make_executor(node, packages, testbed): def executor(commands): @@ -171,10 +189,12 @@ class Experiment(mod.Experiment): if isinstance(self.testbed, docker.Testbed): return - if isinstance(self.testbed, local.Testbed): + if self._is_local: subprocess.check_call('sudo -v'.split()) - self.irmd = subprocess.Popen(["sudo", "irmd"]) - logger.info("Started IRMd, sleeping 2 seconds...") + popen = subprocess.Popen(["sudo", "irmd"]) + self.irmd = Process(None, popen.pid, "irmd", popen=popen) + logger.info("Started IRMd (pid %d), sleeping 2 seconds...", + popen.pid) time.sleep(2) else: for node in self.nodes: @@ -203,8 +223,14 @@ class Experiment(mod.Experiment): m_processing.call_in_parallel(names, args, executors) + @property + def _is_local(self): + """True if the testbed runs locally (single IRMd).""" + return isinstance(self.testbed, + (local.Testbed, localnet.LocalNetworkTestbed)) + def install_ouroboros(self): - if isinstance(self.testbed, local.Testbed): + if self._is_local: return packages = ["cmake", "protobuf-c-compiler", "git", "libfuse-dev", @@ -221,7 +247,7 @@ class Experiment(mod.Experiment): self._install_packages_and_execute_cmds(packages, cmds) def update_ouroboros(self, branch): - if isinstance(self.testbed, local.Testbed): + if self._is_local: return if branch is not None: @@ -255,7 +281,7 @@ class Experiment(mod.Experiment): self._install_packages_and_execute_cmds(packages, []) def install_ouroboros_python_exporter(self): - if isinstance(self.testbed, local.Testbed): + if self._is_local: return if self.influxdb is None: @@ -315,97 +341,106 @@ class Experiment(mod.Experiment): self._install_packages_and_execute_cmds([], commands, [src]) + def _get_n1_ipcp_names(self, node, ipcp): + """Get N-1 IPCP names for name registration.""" + reg_ipcps = [] + if ipcp.layer in node.registrations: + for lower in node.registrations[ipcp.layer]: + for ipcp_b in node.ipcps: + if ipcp_b in lower.ipcps: + reg_ipcps.append(ipcp_b.name) + return reg_ipcps + def create_ipcps(self): for node in self.nodes: - cmds = list() for ipcp in node.ipcps: - cmds2 = list() - if ipcp.dif_bootstrapper: - cmd = "irm i b n " + ipcp.name - else: - cmd = "irm i c n " + ipcp.name - - if isinstance(ipcp.dif, mod.ShimEthDIF): - if isinstance(self.testbed, local.Testbed): - cmd += " type local layer " + ipcp.dif.name - else: - cmd += " type eth-dix dev " + ipcp.ifname - cmd += " layer " + ipcp.dif.name - elif isinstance(ipcp.dif, mod.NormalDIF): - cmd += " type unicast" - if ipcp.dif_bootstrapper: - pols = ipcp.dif.policy.get_policies() - for comp in pols: - for pol in pols[comp]: - cmd += " " + comp + " " + pol - cmd += " layer " + ipcp.dif.name + " autobind" - - cmd2 = "irm n r " + ipcp.name - for dif_b in node.dif_registrations[ipcp.dif]: - for ipcp_b in node.ipcps: - if ipcp_b in dif_b.ipcps: - cmd2 += " ipcp " + ipcp_b.name - cmds2.append(cmd2) - cmd2 = "irm n r " + ipcp.dif.name - for dif_b in node.dif_registrations[ipcp.dif]: - for ipcp_b in node.ipcps: - if ipcp_b in dif_b.ipcps: - cmd2 += " ipcp " + ipcp_b.name - cmds2.append(cmd2) - elif isinstance(ipcp.dif, mod.ShimUDPDIF): - logger.error("UDP IPCP not supported yet") - continue + layer = ipcp.layer + layer_type = layer.layer_type + + # Plain local testbed: use local IPCP type for eth layers + # (LocalNetworkTestbed has real veth interfaces) + if layer.is_eth \ + and isinstance(self.testbed, local.Testbed) \ + and not isinstance(self.testbed, + localnet.LocalNetworkTestbed): + layer_type = LayerType.LOCAL + + if layer.is_shim: + # Shim IPCPs are always bootstrapped + eth_dev = None + ip_addr = None + if layer.is_eth \ + and (not isinstance(self.testbed, local.Testbed) + or isinstance(self.testbed, + localnet.LocalNetworkTestbed)): + eth_dev = ipcp.ifname + if layer.is_udp and hasattr(ipcp, 'ip_addr'): + ip_addr = ipcp.ip_addr + + self.irm.create_ipcp(node, ipcp.name, layer_type) + self.irm.bootstrap_ipcp( + node, ipcp.name, layer_type, + layer.name, eth_dev=eth_dev, ip_addr=ip_addr, + autobind=False) + elif ipcp.bootstrapper: + # Unicast/broadcast bootstrapper + self.irm.create_ipcp(node, ipcp.name, layer_type) + + policies = {} + if hasattr(layer, 'policy'): + pols = layer.policy.get_policies() + for comp, pol_dict in pols.items(): + if pol_dict: + policies[comp] = next(iter(pol_dict)) + + self.irm.bootstrap_ipcp( + node, ipcp.name, layer_type, + layer.name, policies=policies) + + # Postpone N-1 registrations + self.r_ipcps[ipcp] = \ + self._get_n1_ipcp_names(node, ipcp) else: - logger.error("Unsupported IPCP type") - continue + # Non-bootstrapper: just create, will enroll later + self.irm.create_ipcp(node, ipcp.name, layer_type) - cmds.append(cmd) - # Postpone registrations - self.r_ipcps[ipcp] = cmds2 - - node.execute_commands(cmds, time_out=None) - - def enroll_dif(self, el): + def enroll_layer(self, el): for e in el: ipcp = e['enrollee'] - cmds = list() + node = ipcp.node - # Execute postponed registration + # Execute postponed registration for enroller if e['enroller'] in self.r_ipcps: - e['enroller'].node.execute_commands(self.r_ipcps[e['enroller']], - time_out=None) - self.r_ipcps.pop(e['enroller'], None) - - cmd = "irm n r " + ipcp.name - for dif_b in e['enrollee'].node.dif_registrations[ipcp.dif]: - for ipcp_b in e['enrollee'].node.ipcps: - if ipcp_b in dif_b.ipcps: - cmd += " ipcp " + ipcp_b.name - cmds.append(cmd) - cmd = "irm i e n " + ipcp.name + " layer " + e['dif'].name + \ - " autobind" - cmds.append(cmd) - cmd = "irm n r " + ipcp.dif.name - for dif_b in e['enrollee'].node.dif_registrations[ipcp.dif]: - for ipcp_b in e['enrollee'].node.ipcps: - if ipcp_b in dif_b.ipcps: - cmd += " ipcp " + ipcp_b.name - cmds.append(cmd) - - e['enrollee'].node.execute_commands(cmds, time_out=None) + reg_ipcps = self.r_ipcps[e['enroller']] + enroller_node = e['enroller'].node + self.irm.reg_name(enroller_node, e['enroller'].name, + reg_ipcps) + self.irm.reg_name(enroller_node, e['enroller'].layer.name, + reg_ipcps) + del self.r_ipcps[e['enroller']] + + # Register enrollee name in N-1 layers + reg_ipcps = self._get_n1_ipcp_names(node, ipcp) + self.irm.reg_name(node, ipcp.name, reg_ipcps) + + # Enroll + self.irm.enroll_ipcp(node, ipcp.name, e['enroller'].name) + + # Register layer name + self.irm.reg_name(node, ipcp.layer.name, reg_ipcps) def setup_flows(self, el): for e in el: ipcp = e['src'] - cmd = "irm i conn n " + ipcp.name + " dst " + e['dst'].name retry = 0 max_retries = 3 while retry < max_retries: time.sleep(retry * 5) try: - ipcp.node.execute_command(cmd, time_out=None) + self.irm.connect_ipcp( + ipcp.node, ipcp.name, e['dst'].name) break - except Exception as e: + except Exception as ex: retry += 1 logger.error('Failed to connect IPCP, retrying: ' + str(retry) + '/' + str(max_retries) + @@ -422,11 +457,10 @@ class Experiment(mod.Experiment): logger.info("Installed on all nodes...") def _bootstrap_prototype(self): - for dif in self.dif_ordering: - if isinstance(dif, mod.NormalDIF): - if len(dif.qos_cubes) != 0: - logger.warn('QoS cubes not (yet) supported by ' - 'the Ouroboros plugin. Will ignore.') + for layer in self.layer_ordering: + if hasattr(layer, 'qos_cubes') and len(layer.qos_cubes) != 0: + logger.warning('QoS cubes not (yet) supported by ' + 'the Ouroboros plugin. Will ignore.') logger.info("Starting IRMd on all nodes...") self.setup_ouroboros() logger.info("Creating IPCPs") @@ -435,7 +469,7 @@ class Experiment(mod.Experiment): for enrolls, flows in zip(self.enrollments, self.flows): - self.enroll_dif(enrolls) + self.enroll_layer(enrolls) self.setup_flows(flows) logger.info("All done, have fun!") @@ -455,7 +489,8 @@ class Experiment(mod.Experiment): cmds.append(kill + 'ipcpd-broadcast || true') cmds.append(kill + 'ipcpd-eth-llc || true') cmds.append(kill + 'ipcpd-eth-dix || true') - cmds.append(kill + 'ipcpd-udp || true') + cmds.append(kill + 'ipcpd-udp4 || true') + cmds.append(kill + 'ipcpd-udp6 || true') cmds.append(kill + 'ipcpd-local || true') cmds.append(kill + 'ocbr || true') cmds.append(kill + 'oping || true') @@ -472,10 +507,10 @@ class Experiment(mod.Experiment): cmds.append('killall -15 irmd') logger.info("Killing Ouroboros...") - if isinstance(self.testbed, local.Testbed): + if self._is_local: cmds = list(map(lambda c: "sudo %s" % (c,), cmds)) for cmd in cmds: - subprocess.check_call(cmd.split()) + subprocess.call(cmd.split()) else: for node in self.nodes: node.execute_commands(cmds, time_out=None, as_root=True) @@ -486,9 +521,9 @@ class Experiment(mod.Experiment): def _stop_metrics_exporter(self, nodes): self.stop_ouroboros_metrics_exporter(nodes) - def destroy_dif(self, dif): - for ipcp in dif.ipcps: - ipcp.node.execute_command('irm i d n ' + ipcp.name) + def destroy_layer(self, layer): + for ipcp in layer.ipcps: + self.irm.destroy_ipcp(ipcp.node, ipcp.name) def parse_stats(self, lines, spaces=0): d = {} @@ -513,38 +548,38 @@ class Experiment(mod.Experiment): return d - def export_dif_bandwidth(self, filename, dif): + def export_layer_bandwidth(self, filename, layer): f = open(filename, 'w') - for node in dif.members: - ipcp = node.get_ipcp_by_dif(dif) + for node in layer.members: + ipcp = node.get_ipcp_by_layer(layer) # Get IPCP address if not hasattr(ipcp, 'address'): path = '/tmp/ouroboros/' + ipcp.name + '/dt*' - dt_path = node.execute_command('ls -d %s' % path) + dt_path = node.execute_command('ls -d %s' % path).stdout dts = dt_path.split('.') ipcp.address = int(dts[-1]) logger.info('IPCP %s has dt component ' 'with address %d' % (ipcp.name, ipcp.address)) - for node in dif.members: - ipcp = node.get_ipcp_by_dif(dif) + for node in layer.members: + ipcp = node.get_ipcp_by_layer(layer) dt_path = '/tmp/ouroboros/' + ipcp.name + '/dt.' + \ str(ipcp.address) + '/' # Get flows to other endpoints - fd = node.execute_command('ls --ignore=[01] %s' % dt_path) + fd = node.execute_command('ls --ignore=[01] %s' % dt_path).stdout fds = fd.split('\n') for fd in fds: fd_path = dt_path + fd - fd_file = node.execute_command('cat %s' % fd_path) + fd_file = node.execute_command('cat %s' % fd_path).stdout d = self.parse_stats(fd_file.splitlines()) remote = d["Endpoint address"] ipcp2_name = '' - for ipcp2 in dif.ipcps: + for ipcp2 in layer.ipcps: if ipcp2.address == int(remote): ipcp2_name = ipcp2.name diff --git a/rumba/prototypes/rlite.py b/rumba/prototypes/rlite.py deleted file mode 100644 index e67e539..0000000 --- a/rumba/prototypes/rlite.py +++ /dev/null @@ -1,194 +0,0 @@ -# -# A library to manage ARCFIRE experiments -# -# Copyright (C) 2017-2018 Nextworks S.r.l. -# Copyright (C) 2017-2018 imec -# -# Sander Vrijders <sander.vrijders@ugent.be> -# Dimitri Staessens <dimitri.staessens@ugent.be> -# Vincenzo Maffione <v.maffione@nextworks.it> -# Marco Capitani <m.capitani@nextworks.it> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., http://www.fsf.org/about/contact/. -# - -import rumba.ssh_support as ssh -import rumba.model as mod -import rumba.log as log -import rumba.multiprocess as m_processing - -import time - - -logger = log.get_logger(__name__) - - -class Experiment(mod.Experiment): - """ - Represents an rlite experiment. - """ - - def __init__(self, testbed, nodes=None, - git_repo='https://gitlab.com/arcfire/rlite', - git_branch='master', enrollment_strategy='minimal'): - """ - Initializes the experiment class. - - :param testbed: The testbed to run the experiment on. - :param nodes: The list of nodes. - :param git_repo: The git repository to use for installation. - :param git_branch: The branch of the git repository to use. - :param enrollment_strategy: Can be 'full-mesh', 'minimal' or 'manual'. - """ - mod.Experiment.__init__(self, testbed, nodes, git_repo, git_branch, - prototype_logs=['/tmp/uipcp.log'], - enrollment_strategy=enrollment_strategy) - - @staticmethod - def make_executor(node, packages, testbed): - def executor(commands): - ssh.aptitude_install(testbed, node, packages) - node.execute_commands(commands, time_out=None, use_proxy=True) - return executor - - def prototype_name(self): - return 'rlite' - - def execute_commands(self, node, cmds): - ssh.execute_commands(self.testbed, node.ssh_config, - cmds, time_out=None) - - # Prepend sudo to all commands if the user is not 'root' - def may_sudo(self, cmds): - if self.testbed.username != 'root': - for i in range(len(cmds)): - cmds[i] = "sudo %s" % cmds[i] - - def init_nodes(self): - # Load kernel modules and start the uipcps daemon - cmds = ["modprobe rlite", - "modprobe rlite-normal", - "modprobe rlite-shim-eth", - "modprobe rlite-shim-udp4", - "modprobe rlite-shim-loopback", - "rlite-uipcps -v DBG > /tmp/uipcp.log 2>&1 &"] - self.may_sudo(cmds) - - for node in self.nodes: - self.execute_commands(node, cmds) - - def create_ipcps(self): - for node in self.nodes: - cmds = [] - - for ipcp in node.ipcps: - # Generate the command to create the IPCP - if isinstance(ipcp.dif, mod.NormalDIF): - ipcp_type = 'normal' - elif isinstance(ipcp.dif, mod.ShimEthDIF): - ipcp_type = 'shim-eth' - elif isinstance(ipcp.dif, mod.ShimUDPDIF): - ipcp_type = 'shim-udp4' - else: - logger.warning( - "unknown type for DIF %s, default to loopback", - ipcp.dif.name) - ipcp_type = 'shim-loopback' - - cmds.append("rlite-ctl ipcp-create %s %s %s" % - (ipcp.name, ipcp_type, ipcp.dif.name)) - - # Generate the command to configure the interface - # name for the shim-eth - if isinstance(ipcp.dif, mod.ShimEthDIF): - cmds.append("rlite-ctl ipcp-config %s netdev %s" - % (ipcp.name, ipcp.ifname)) - - if isinstance(ipcp.dif, mod.NormalDIF) \ - and ipcp.dif_bootstrapper: - cmds.append("rlite-ctl ipcp-enroller-enable %s" - % (ipcp.name)) - - self.may_sudo(cmds) - self.execute_commands(node, cmds) - - def register_ipcps(self): - for node in self.nodes: - cmds = [] - - for ipcp in node.ipcps: - for lower in ipcp.registrations: - cmds.append("rlite-ctl ipcp-register %s %s" - % (ipcp.name, lower.name)) - - self.may_sudo(cmds) - self.execute_commands(node, cmds) - - def enroll_ipcps(self): - for el in self.enrollments: - for e in el: - d = {'enrollee': e['enrollee'].name, - 'dif': e['dif'].name, - 'lower_dif': e['lower_dif'].name, - 'enroller': e['enroller'].name - } - cmd = "rlite-ctl ipcp-enroll-retry %(enrollee)s %(dif)s "\ - "%(lower_dif)s %(enroller)s" % d - cmds = [cmd] - self.may_sudo(cmds) - self.execute_commands(e['enrollee'].node, cmds) - time.sleep(1) - - def _install_prototype(self): - logger.info("installing rlite on all nodes") - - packages = ["g++", "gcc", "cmake", "linux-headers-$(uname -r)", - "protobuf-compiler", "libprotobuf-dev", "git"] - - cmds = ["rm -rf ~/rlite", - "cd ~; git clone -b " + self.git_branch + " " + self.git_repo, - "cd ~/rlite && ./configure && make && sudo make install", - "cd ~/rlite && sudo make depmod"] - - names = [] - executors = [] - args = [] - - for node in self.nodes: - executor = self.make_executor(node, packages, self.testbed) - names.append(node.name) - executors.append(executor) - args.append(cmds) - m_processing.call_in_parallel(names, args, executors) - logger.info("installation complete") - - def _bootstrap_prototype(self): - for dif in self.dif_ordering: - if isinstance(dif, mod.NormalDIF): - if len(dif.qos_cubes) != 0: - logger.warn('QoS cubes not (yet) supported by ' - 'the rlite plugin. Will ignore.') - logger.info("setting up") - self.init_nodes() - logger.info("software initialized on all nodes") - self.create_ipcps() - logger.info("IPCPs created on all nodes") - self.register_ipcps() - logger.info("IPCPs registered to their lower DIFs on all nodes") - self.enroll_ipcps() - logger.info("enrollment completed in all DIFs") - - def _terminate_prototype(self): - return diff --git a/rumba/recpoisson.py b/rumba/recpoisson.py index a4cf492..98960a4 100644 --- a/rumba/recpoisson.py +++ b/rumba/recpoisson.py @@ -27,13 +27,7 @@ import math import random -import sys - -if sys.version_info < (3, 2): - from repoze.lru import lru_cache - # from functools32 import lru_cache -else: - from functools import lru_cache +from functools import lru_cache @lru_cache(1000) diff --git a/rumba/ssh_support.py b/rumba/ssh_support.py index d77b59f..d17895a 100644 --- a/rumba/ssh_support.py +++ b/rumba/ssh_support.py @@ -26,7 +26,6 @@ import os import paramiko -import re import time from paramiko.ssh_exception import * @@ -34,20 +33,14 @@ from paramiko.ssh_exception import * import rumba.log as log # Fix input reordering -from rumba.model import Executor +import builtins -try: - import builtins # Only in Python 3 +from rumba.model import Executor - def input(prompt=''): - log.flush_log() - return builtins.input(prompt) -except ImportError: # We are in Python 2 - import __builtin__ - def input(prompt=''): - log.flush_log() - return __builtin__.raw_input(prompt) +def input(prompt=''): + log.flush_log() + return builtins.input(prompt) logger = log.get_logger(__name__) @@ -65,18 +58,16 @@ def get_ssh_client(): return ssh_client -def _print_stream(stream): - o = str(stream.read()).strip('b') - o = o.strip('\'\"') - o = o.rstrip() - o = re.sub(r'(\\n)*$', '', o) - if o != "": - o_array = o.split('\\n') - for oi in o_array: - logger.debug(oi) - else: - o_array = [] - return '\n'.join(o_array) +def _read_stream(stream): + """Read and decode a paramiko channel stream.""" + data = stream.read() + if isinstance(data, bytes): + data = data.decode('utf-8', errors='replace') + text = data.rstrip() + if text: + for line in text.split('\n'): + logger.debug(line) + return text def ssh_connect(hostname, port, username, password, time_out, proxy_server): @@ -162,7 +153,8 @@ def ssh_sftp(ssh_config, testbed): chan.invoke_subsystem("sftp") return paramiko.sftp_client.SFTPClient(chan) -def execute_commands(testbed, ssh_config, commands, time_out=3): +def execute_commands(testbed, ssh_config, commands, time_out=3, + node_name='', check=True): """ Remote execution of a list of shell command on hostname. By default this function will exit (timeout) after 3 seconds. @@ -173,45 +165,51 @@ def execute_commands(testbed, ssh_config, commands, time_out=3): @param time_out: time_out value in seconds, error will be generated if no result received in given number of seconds, the value None can be used when no timeout is needed + @param node_name: name of the node (for CommandResult) + @param check: if True, raise CommandError on non-zero exit code + @return: CommandResult for the last command """ + from rumba.command import CommandResult ssh_connect_check(ssh_config, testbed, time_out) - o = "" + result = None for command in commands: logger.debug("%s@%s:%s >> %s" % (testbed.username, ssh_config.hostname, ssh_config.port, command)) envars = '. /etc/profile;' - command = envars + ' ' + command + full_cmd = envars + ' ' + command chan = ssh_chan(ssh_config, testbed, time_out) stdout = chan.makefile() stderr = chan.makefile_stderr() try: - chan.exec_command(command) + chan.exec_command(full_cmd) except paramiko.ssh_exception.SSHException as e: raise SSHException('Failed to execute command') - o = _print_stream(stdout) - if chan.recv_exit_status() != 0: - # Get ready for printing stdout and stderr - if o != "": - list_print = ['**** STDOUT:'] - list_print += o.split('\\n') - else: - list_print = [] - e = _print_stream(stderr) - if e != "": - list_print.append('**** STDERR:') - list_print += e.split('\\n') - raise SSHException('A remote command returned an error. ' - 'Output:\n\n\t' + - '\n\t'.join(list_print) + '\n') - return o - - -def execute_command(testbed, ssh_config, command, time_out=3): + + out = _read_stream(stdout) + err = _read_stream(stderr) + exit_code = chan.recv_exit_status() + + result = CommandResult( + exit_code=exit_code, + stdout=out, + stderr=err, + command=command, + node=node_name + ) + + if check: + result.check() + + return result + + +def execute_command(testbed, ssh_config, command, time_out=3, + node_name='', check=True): """ Remote execution of a list of shell command on hostname. By default this function will exit (timeout) after 3 seconds. @@ -222,12 +220,13 @@ def execute_command(testbed, ssh_config, command, time_out=3): @param time_out: time_out value in seconds, error will be generated if no result received in given number of seconds, the value None can be used when no timeout is needed + @param node_name: name of the node (for CommandResult) + @param check: if True, raise CommandError on non-zero exit code - @return: stdout resulting from the command + @return: CommandResult from the command """ - o = execute_commands(testbed, ssh_config, [command], time_out) - if o is not None: - return o + return execute_commands(testbed, ssh_config, [command], time_out, + node_name=node_name, check=check) def write_text_to_file(testbed, ssh_config, text, file_name): diff --git a/rumba/storyboard.py b/rumba/storyboard.py index 59fbf6f..f82ae7a 100644 --- a/rumba/storyboard.py +++ b/rumba/storyboard.py @@ -262,8 +262,9 @@ class ClientProcess(_SBEntity): self.running = True try: - self.pid = self.node.execute_command(cmd, as_root=self.as_root) - except ssh_support.SSHException: + result = self.node.execute_command(cmd, as_root=self.as_root) + self.pid = result.stdout + except Exception: logger.warning('Could not start client %s on node %s.', self.ap_id, self.node.name) logger.debug('Client app %s on node %s got pid %s.', @@ -272,8 +273,8 @@ class ClientProcess(_SBEntity): def run_async(self): """Starts this process asynchronously""" - def callback(pid): - self.pid = pid + def callback(result): + self.pid = result.stdout logger.debug('Client app %s on node %s got pid %s.', self.ap_id, self.node.name, self.pid) @@ -325,7 +326,7 @@ class Server(_SBEntity): def __init__(self, ap, arrival_rate, mean_duration, options=None, max_clients=float('inf'), clients=None, nodes=None, min_duration=2, - s_id=None, as_root=False, difs=None): + s_id=None, as_root=False, layers=None): """ :param ap: the application binary or command which should be run @@ -355,10 +356,10 @@ class Server(_SBEntity): :param as_root: if true, the server app will be started with root permissions :type as_root: `bool` - :param difs: the difs this server intends to register to - (note: the effect of this parameter is prototype - dependent, and other strategies might be required) - :type difs: `rumba.model.DIF` or `list` thereof + :param layers: the layers this server intends to register to + (note: the effect of this parameter is prototype + dependent, and other strategies might be required) + :type layers: `rumba.model.Layer` or `list` thereof """ self.ap = ap e_id = s_id if s_id is not None else self.ap.replace(' ', '_') @@ -377,13 +378,13 @@ class Server(_SBEntity): self.pids = {} self.min_duration = min_duration self.as_root = as_root - if difs is None: - difs = [] - elif hasattr(difs, '__iter__'): - difs = list(difs) + if layers is None: + layers = [] + elif hasattr(layers, '__iter__'): + layers = list(layers) else: - difs = [difs] - self.difs = difs + layers = [layers] + self.layers = layers def add_client(self, client): """ @@ -503,8 +504,8 @@ class Server(_SBEntity): self.id, node.name ) - def callback(pid): - self.pids[node] = pid + def callback(result): + self.pids[node] = result.stdout _execute_command(node, cmd, callback, as_root=self.as_root) @@ -568,7 +569,7 @@ class StoryBoard(_SBEntity): self.commands_list = {} self.node_map = {} self.shims = {} - self.difs = {} + self.layers = {} self._build_nodes_lists() # The following must be last, because it needs the info from # _build_nodes_list @@ -585,10 +586,10 @@ class StoryBoard(_SBEntity): if self.experiment is not None: for node in self.experiment.nodes: self.node_map[node.name] = node - for dif in self.experiment.dif_ordering: - self.difs[dif.name] = dif - if isinstance(dif, model.ShimEthDIF): - self.shims[dif.name] = dif + for layer in self.experiment.layer_ordering: + self.layers[layer.name] = layer + if layer.is_eth: + self.shims[layer.name] = layer def _validate_and_add_server(self, s, n=None): if self.experiment is None: @@ -1063,79 +1064,79 @@ class StoryBoard(_SBEntity): dst_dir = os.path.join(local_dir, node.name) if not os.path.isdir(dst_dir): os.mkdir(dst_dir) - logs_list = node.execute_command('ls /tmp/*.rumba.log ' - '|| echo ""') - logs_list = [x for x in logs_list.split('\n') if x != ''] + result = node.execute_command('ls /tmp/*.rumba.log ' + '|| echo ""') + logs_list = [x for x in result.stdout.split('\n') if x != ''] logger.debug('Log list is:\n%s', logs_list) node.fetch_files(logs_list, dst_dir) - def schedule_export_dif_bandwidth(self, t, filename, dif): + def schedule_export_layer_bandwidth(self, t, filename, layer): """ Schedules the generation of a csv file of the bandwidth used by - flows in a certain DIF at a certain time. + flows in a certain layer at a certain time. :param filename: The output csv filename. - :param dif: The DIF to export + :param layer: The layer to export """ if self.experiment is None: raise ValueError("An experiment is needed to schedule commands.") - action = functools.partial(self.experiment.export_dif_bandwidth, - filename, dif) + action = functools.partial(self.experiment.export_layer_bandwidth, + filename, layer) self.add_event(Event(action, ev_time=t)) - def schedule_link_state(self, t, dif, state): + def schedule_link_state(self, t, layer, state): """ - Schedules a link's (`rumba.model.ShimEthDIF`) state to go + Schedules a link's (`rumba.model.EthLayer`) state to go up or down at the specified time. :param t: the time in the storyboard at which the state change should happen :type t: `float` - :param dif: the DIF which should be reconfigured - :type dif: `rumba.model.ShimEthDIF` + :param layer: the layer which should be reconfigured + :type layer: `rumba.model.EthLayer` :param state: the desired state :type state: `str` -- either `up` or `down` """ if self.experiment is None: raise ValueError("An experiment is needed to schedule commands.") - if not isinstance(dif, model.ShimEthDIF): - raise ValueError("Not a Shim Ethernet DIF.") + if not hasattr(layer, 'is_eth') or not layer.is_eth: + raise ValueError("Not an Ethernet layer.") if state not in ['up', 'down']: raise ValueError('Only possible states are "up" and "down"') if self._script is None: self._script = _Script(self) - for node in dif.members: - action = functools.partial(node.set_link_state, dif, state) + for node in layer.members: + action = functools.partial(node.set_link_state, layer, state) self._script.add_event(Event(action, ev_time=t)) - def schedule_link_up(self, t, dif): + def schedule_link_up(self, t, layer): """ - Schedules a link's (`rumba.model.ShimEthDIF`) state to go + Schedules a link's (`rumba.model.EthLayer`) state to go up at the specified time. :param t: the time in the storyboard at which the state change should happen :type t: `float` - :param dif: the DIF which should be reconfigured - :type dif: `rumba.model.ShimEthDIF` + :param layer: the layer which should be reconfigured + :type layer: `rumba.model.EthLayer` """ - self.schedule_link_state(t, dif, 'up') + self.schedule_link_state(t, layer, 'up') - def schedule_link_down(self, t, dif): + def schedule_link_down(self, t, layer): """ - Schedules a link's (`rumba.model.ShimEthDIF`) state to go + Schedules a link's (`rumba.model.EthLayer`) state to go down at the specified time. :param t: the time in the storyboard at which the state change should happen :type t: `float` - :param dif: the DIF which should be reconfigured - :type dif: `rumba.model.ShimEthDIF` + :param layer: the layer which should be reconfigured + :type layer: `rumba.model.EthLayer` """ - self.schedule_link_state(t, dif, 'down') + self.schedule_link_state(t, layer, 'down') def schedule_node_state(self, t, node, state): """ @@ -1157,10 +1158,10 @@ class StoryBoard(_SBEntity): if self._script is None: self._script = _Script(self) - for dif in node.difs: - if not isinstance(dif, model.ShimEthDIF): + for layer in node.layers: + if not layer.is_eth: continue - action = functools.partial(node.set_link_state, dif, state) + action = functools.partial(node.set_link_state, layer, state) self._script.add_event(Event(action, ev_time=t)) def schedule_node_up(self, t, node): @@ -1189,20 +1190,20 @@ class StoryBoard(_SBEntity): """ self.schedule_node_state(t, node, 'down') - def schedule_destroy_dif(self, t, dif): + def schedule_destroy_layer(self, t, layer): """ - Destroys a DIF at the specified time. + Destroys a layer at the specified time. :param t: the time in the storyboard at which the state change should happen :type t: `float` - :param dif: the DIF which should go down - :type dif: `rumba.model.DIF` + :param layer: the layer which should go down + :type layer: `rumba.model.Layer` """ if self.experiment is None: raise ValueError("An experiment is needed to schedule commands.") - action = functools.partial(self.experiment.destroy_dif, dif) + action = functools.partial(self.experiment.destroy_layer, layer) self.add_event(Event(action, ev_time=t)) def write_script(self, buffer): @@ -1283,7 +1284,7 @@ class StoryBoard(_SBEntity): buffer = StringIO(string) self.parse_script(buffer, clean) - def capture_traffic(self, start, end, node, dif): + def capture_traffic(self, start, end, node, layer): """ Captures the traffic of an interface on a node. @@ -1293,11 +1294,11 @@ class StoryBoard(_SBEntity): :type end: `float` :param node: the node to capture on. :type node: `rumba.model.Node` - :param dif: the node's Shim Ethernet DIF whose interface - will be used for the capture. - :type dif: `rumba.model.ShimEthDIF` + :param layer: the node's Ethernet layer whose interface + will be used for the capture. + :type layer: `rumba.model.EthLayer` """ - for ipcp in dif.ipcps: + for ipcp in layer.ipcps: if ipcp.node is not node: continue # In case tcpdump is not present, this assumes a testbed @@ -1311,7 +1312,7 @@ class StoryBoard(_SBEntity): pcap_file = ( node.name + '_' + - dif.name + + layer.name + '_' + str(uuid.uuid4())[0:4] + ".pcap" ) @@ -1533,8 +1534,10 @@ class _Script(object): 'Server': self._servers, 'Client': self._clients, 'ClientProcess': self._processes, - 'ShimEthDIF': self._shims, - 'DIF': self._storyboard.difs + 'EthLayer': self._shims, + 'EthDixLayer': self._shims, + 'EthLlcLayer': self._shims, + 'Layer': self._storyboard.layers } def add_process(self, process): diff --git a/rumba/testbeds/dockertb.py b/rumba/testbeds/dockertb.py index d2a1334..8323bdc 100644 --- a/rumba/testbeds/dockertb.py +++ b/rumba/testbeds/dockertb.py @@ -70,10 +70,7 @@ class Testbed(mod.Testbed): docker_client = self.docker_client if not self.base_image: - if experiment.prototype_name() == 'ouroboros': - self.base_image = 'arcfirerumba/ouroboros' - else: - raise Exception('Only Ouroboros supported for now.') + self.base_image = 'arcfirerumba/ouroboros' img = self.base_image.rsplit(":", 1) @@ -107,8 +104,8 @@ class Testbed(mod.Testbed): if not os.path.exists("/var/run/netns"): subprocess.check_call(cmd.split()) - for shim in experiment.dif_ordering: - if not isinstance(shim, mod.ShimEthDIF): + for shim in experiment.layer_ordering: + if not shim.is_eth: # Nothing to do here continue @@ -151,7 +148,7 @@ class Testbed(mod.Testbed): subprocess.check_call(cmd.split()) for ipcp in node.ipcps: - if isinstance(ipcp, mod.ShimEthIPCP): + if isinstance(ipcp, mod.EthIPCP): if ipcp.ifname is None: ipcp.ifname = "e%i" % node.ipcps.index(ipcp) @@ -168,15 +165,15 @@ class Testbed(mod.Testbed): cmd = "" if self.use_ovs: - cmd += ('sudo ovs-vsctl add-port %(dif)s %(node)s.%(' + cmd += ('sudo ovs-vsctl add-port %(layer)s %(node)s.%(' 'ifname)s') else: cmd += ('sudo ip link set %(node)s.%(ifname)s master ' - '%(dif)s') + '%(layer)s') cmd = (cmd % {'node': node_name, 'ifname': ipcp.ifname, - 'dif': ipcp.dif.name}) + 'layer': ipcp.layer.name}) logger.debug('executing >> %s', cmd) @@ -212,9 +209,8 @@ class Testbed(mod.Testbed): logger.debug('Stopping node %s' % name) container.remove(force=True) - for shim in experiment.dif_ordering: - if isinstance(shim, mod.ShimEthDIF) \ - and shim.name in self.active_bridges: + for shim in experiment.layer_ordering: + if shim.is_eth and shim.name in self.active_bridges: cmd = "" if self.use_ovs: cmd += 'sudo ovs-vsctl del-br %(shim)s' diff --git a/rumba/testbeds/emulab.py b/rumba/testbeds/emulab.py index 8cd31cf..d0b36b4 100644 --- a/rumba/testbeds/emulab.py +++ b/rumba/testbeds/emulab.py @@ -183,14 +183,14 @@ class Testbed(mod.Testbed): ns2_script += "tb-set-node-os $" + node.name + " " + \ self.image + "\n" - for dif in experiment.dif_ordering: - if isinstance(dif, mod.ShimEthDIF): - if len(dif.ipcps) != 2: + for layer in experiment.layer_ordering: + if layer.is_eth: + if len(layer.ipcps) != 2: continue - ns2_script += "set " + dif.name + \ + ns2_script += "set " + layer.name + \ " [$ns duplex-link $" + \ - dif.members[0].name + " $" + \ - dif.members[1].name + " 1000Mb 0ms DropTail]\n" + layer.members[0].name + " $" + \ + layer.members[1].name + " 1000Mb 0ms DropTail]\n" ns2_script += "$ns run\n" @@ -248,7 +248,7 @@ class Testbed(mod.Testbed): if node.name != node_name: continue for ipcp in node.ipcps: - if ipcp.dif.name == link_name: + if ipcp.layer.name == link_name: self.ip[ipcp] = link_ip for node in experiment.nodes: @@ -258,7 +258,7 @@ class Testbed(mod.Testbed): for item in output: item = item.split() for ipcp in node.ipcps: - if isinstance(ipcp, mod.ShimEthIPCP): + if isinstance(ipcp, mod.EthIPCP): if self.ip[ipcp] == item[1]: ipcp.ifname = item[0] diff --git a/rumba/testbeds/jfed.py b/rumba/testbeds/jfed.py index e9e281d..a06aa50 100644 --- a/rumba/testbeds/jfed.py +++ b/rumba/testbeds/jfed.py @@ -29,14 +29,10 @@ import getpass import xml.dom.minidom as xml import os.path import tarfile -import sys from rumba.executors.ssh import SSHExecutor -if sys.version_info[0] >= 3: - from urllib.request import urlretrieve -else: - from urllib import urlretrieve +from urllib.request import urlretrieve import rumba.model as mod import rumba.log as log @@ -202,24 +198,24 @@ class Testbed(mod.Testbed): node.ifs = 0 for ipcp in node.ipcps: - if isinstance(ipcp, mod.ShimEthIPCP): + if isinstance(ipcp, mod.EthIPCP): el3 = doc.createElement("interface") self.if_id[ipcp] = node.name + ":if" + str(node.ifs) el3.setAttribute("client_id", self.if_id[ipcp]) node.ifs += 1 el.appendChild(el3) - for dif in experiment.dif_ordering: - if isinstance(dif, mod.ShimEthDIF): + for layer in experiment.layer_ordering: + if layer.is_eth: el = doc.createElement("link") top_el.appendChild(el) - el.setAttribute("client_id", dif.name) + el.setAttribute("client_id", layer.name) el2 = doc.createElement("component_manager") el2.setAttribute("name", self.authority) el.appendChild(el2) - for ipcp in dif.ipcps: + for ipcp in layer.ipcps: el3 = doc.createElement("interface_ref") el3.setAttribute("client_id", self.if_id[ipcp]) el.appendChild(el3) @@ -370,7 +366,7 @@ class Testbed(mod.Testbed): ) i_name = intf.getAttribute("client_id") for ipcp in node_n.ipcps: - if isinstance(ipcp, mod.ShimEthIPCP): + if isinstance(ipcp, mod.EthIPCP): if self.if_id[ipcp] == i_name: ipcp.ifname = ifname if ifname is None or ifname == "": diff --git a/rumba/testbeds/localnet.py b/rumba/testbeds/localnet.py new file mode 100644 index 0000000..95a6bdc --- /dev/null +++ b/rumba/testbeds/localnet.py @@ -0,0 +1,257 @@ +# +# Rumba - Local Network Testbed (veth + bridges) +# +# Copyright (C) 2017-2026 imec +# +# Dimitri Staessens <dimitri.staessens@ugent.be> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., http://www.fsf.org/about/contact/. +# + +import hashlib +import subprocess +import tempfile + +import rumba.model as mod +import rumba.log as log + +from rumba.executors.local import LocalExecutor + +logger = log.get_logger(__name__) + + +def _run(cmd): + """Run a command with sudo, logging it first.""" + logger.debug('executing >> %s', cmd) + subprocess.check_call(cmd.split()) + + +class LocalNetworkTestbed(mod.Testbed): + """ + Local testbed using Linux bridges and veth pairs. + + Each Ethernet layer gets a Linux bridge. Each node's Ethernet IPCP + gets a veth pair: one end is attached to the layer's bridge, the + other end is the interface that the IPCP binds to. All processes + (IRMd, IPCPs, applications) run in the default network namespace + under a single IRMd instance. + + This allows testing the real eth-dix / eth-llc stack locally with + netem support on the veth interfaces. + """ + + def __init__(self, exp_name='localnet', username='', + proj_name='rumba', password=''): + """ + :param exp_name: The experiment name. + :param username: User of the experiment. + :param proj_name: Project name of the experiment. + :param password: Password of the user. + """ + mod.Testbed.__init__(self, exp_name, username, password, proj_name) + + self.executor = LocalExecutor(self) + self.active_bridges = set() + self.active_veths = [] # list of (host_end, ipcp_end) tuples + self._hosts_backup = None # original /etc/hosts content + + def _swap_in(self, experiment): + # Ensure sudo is primed + subprocess.check_call('sudo -v'.split()) + + # Create a bridge for each Ethernet layer + for layer in experiment.layer_ordering: + if not layer.is_eth: + continue + + bridge = 'br-%s' % layer.name + _run('sudo ip link add %s type bridge' % bridge) + _run('sudo ip link set dev %s up' % bridge) + self.active_bridges.add(bridge) + + # Create veth pairs for each Ethernet IPCP + veth_idx = 0 + for node in experiment.nodes: + for ipcp in node.ipcps: + if not isinstance(ipcp, mod.EthIPCP): + continue + + bridge = 'br-%s' % ipcp.layer.name + + # Generate unique interface names (max 15 chars) + ipcp_end = 'veth%d' % veth_idx + host_end = 'vb%d' % veth_idx + veth_idx += 1 + + # Assign the IPCP-facing interface name + ipcp.ifname = ipcp_end + + # Create veth pair + _run('sudo ip link add %s type veth peer name %s' + % (host_end, ipcp_end)) + + # Attach bridge end to the layer bridge + _run('sudo ip link set %s master %s' % (host_end, bridge)) + + # Bring both ends up + _run('sudo ip link set dev %s up' % host_end) + _run('sudo ip link set dev %s up' % ipcp_end) + + self.active_veths.append((host_end, ipcp_end)) + + # Create veth pairs with IP addresses for each UDP layer. + # UDP layers are point-to-point (max 2 members), so we create + # a single veth pair and assign an IP to each end. + udp_layers_done = set() + udp_layer_idx = 0 + for layer in experiment.layer_ordering: + if not layer.is_udp: + continue + if layer.name in udp_layers_done: + continue + udp_layers_done.add(layer.name) + udp_layer_idx += 1 + + # Collect the (at most 2) UdpIPCPs for this layer + udp_ipcps = [ipcp for node in experiment.nodes + for ipcp in node.ipcps + if isinstance(ipcp, mod.UdpIPCP) + and ipcp.layer == layer] + + if len(udp_ipcps) != 2: + raise Exception( + "UDP layer %s needs exactly 2 members, got %d" + % (layer.name, len(udp_ipcps))) + + end_a = 'veth%d' % veth_idx + end_b = 'veth%d' % (veth_idx + 1) + veth_idx += 2 + + _run('sudo ip link add %s type veth peer name %s' + % (end_a, end_b)) + + # Assign IP addresses (10.{layer_idx}.0.{1,2}/24) + ip_a = '10.%d.0.1/24' % udp_layer_idx + ip_b = '10.%d.0.2/24' % udp_layer_idx + + _run('sudo ip addr add %s dev %s' % (ip_a, end_a)) + _run('sudo ip addr add %s dev %s' % (ip_b, end_b)) + _run('sudo ip link set dev %s up' % end_a) + _run('sudo ip link set dev %s up' % end_b) + + udp_ipcps[0].ifname = end_a + udp_ipcps[0].ip_addr = '10.%d.0.1' % udp_layer_idx + udp_ipcps[1].ifname = end_b + udp_ipcps[1].ip_addr = '10.%d.0.2' % udp_layer_idx + + self.active_veths.append((end_a, end_b)) + + # ---- /etc/hosts entries for UDP name resolution ---- + # The UDP shim IPCP resolves names via DNS (falling back to + # /etc/hosts when dns_addr is 0.0.0.0). We add entries so + # that IPCP names and overlay names registered in UDP layers + # resolve to the correct veth IP address. + hosts_entries = [] # list of (ip, md5_hash) tuples + for node in experiment.nodes: + # Find this node's UDP IPCP(s) and their IPs + for ipcp in node.ipcps: + if not isinstance(ipcp, mod.UdpIPCP): + continue + if ipcp.ip_addr is None or ipcp.ip_addr == '0.0.0.0': + continue + udp_ip = ipcp.ip_addr + udp_layer = ipcp.layer + + # 1) The UDP IPCP's own name (MD5-hashed) + hosts_entries.append((udp_ip, ipcp.name)) + + # 2) Names of overlay IPCPs registered in this UDP layer, + # plus their layer names (needed for enrollment) + for other_ipcp in node.ipcps: + if other_ipcp == ipcp: + continue + if other_ipcp.layer in node.registrations \ + and udp_layer in node.registrations[ + other_ipcp.layer]: + hosts_entries.append((udp_ip, other_ipcp.name)) + hosts_entries.append((udp_ip, + other_ipcp.layer.name)) + + if hosts_entries: + result = subprocess.run( + ['cat', '/etc/hosts'], capture_output=True, text=True) + self._hosts_backup = result.stdout + + # Deduplicate and append. + # The UDP IPCP resolves names by their MD5 hex digest, + # so /etc/hosts must map the hash to the IP address. + seen = set() + lines = [] + for ip, name in hosts_entries: + md5 = hashlib.md5(name.encode()).hexdigest() + key = (ip, md5) + if key not in seen: + seen.add(key) + lines.append('%s %s # %s' % (ip, md5, name)) + + block = '\n# --- rumba localnet UDP ---\n' + for line in lines: + block += line + '\n' + + with tempfile.NamedTemporaryFile( + mode='w', suffix='.hosts', delete=False) as tmp: + tmp.write(self._hosts_backup + block) + tmp_path = tmp.name + _run('sudo cp %s /etc/hosts' % tmp_path) + subprocess.run(['rm', '-f', tmp_path], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + + logger.debug('/etc/hosts entries added: %s', lines) + + logger.info("Experiment swapped in (%d bridges, %d veth pairs)", + len(self.active_bridges), len(self.active_veths)) + + def _swap_out(self, experiment): + # Restore /etc/hosts if we modified it + if self._hosts_backup is not None: + with tempfile.NamedTemporaryFile( + mode='w', suffix='.hosts', delete=False) as tmp: + tmp.write(self._hosts_backup) + tmp_path = tmp.name + _run('sudo cp %s /etc/hosts' % tmp_path) + subprocess.run(['rm', '-f', tmp_path], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + self._hosts_backup = None + logger.debug('/etc/hosts restored') + + # Deleting one end of a veth pair automatically removes the other + for host_end, _ in self.active_veths: + try: + _run('sudo ip link del %s' % host_end) + except subprocess.CalledProcessError: + logger.debug('veth %s already gone', host_end) + + self.active_veths = [] + + for bridge in list(self.active_bridges): + try: + _run('sudo ip link del %s' % bridge) + except subprocess.CalledProcessError: + logger.debug('bridge %s already gone', bridge) + self.active_bridges.discard(bridge) + + logger.info("Experiment swapped out") diff --git a/rumba/testbeds/qemu.py b/rumba/testbeds/qemu.py deleted file mode 100644 index 53a3e42..0000000 --- a/rumba/testbeds/qemu.py +++ /dev/null @@ -1,457 +0,0 @@ -# -# A library to manage ARCFIRE experiments -# -# Copyright (C) 2017-2018 Nextworks S.r.l. -# Copyright (C) 2017-2018 imec -# -# Sander Vrijders <sander.vrijders@ugent.be> -# Dimitri Staessens <dimitri.staessens@ugent.be> -# Vincenzo Maffione <v.maffione@nextworks.it> -# Marco Capitani <m.capitani@nextworks.it> -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., http://www.fsf.org/about/contact/. -# - -import multiprocessing as old_m -import os -import subprocess -import sys -import time -from subprocess import CalledProcessError - -import rumba.model as mod -import rumba.log as log -import rumba.multiprocess as m_processing - -from rumba.executors.ssh import SSHExecutor -from rumba.ssh_support import SSHException - -if sys.version_info[0] >= 3: - from urllib.request import urlretrieve -else: - from urllib import urlretrieve - - -logger = log.get_logger(__name__) - -USER_HOME = os.path.expanduser("~") - - -class Testbed(mod.Testbed): - """ - Represents a QEMU testbed. - """ - def __init__(self, exp_name='foo', bzimage_path=None, initramfs_path=None, - proj_name="rumba", password="root", username="root", - public_key_path=os.path.join(USER_HOME, '.ssh', 'id_rsa.pub'), - use_vhost=True): - """ - Initializes the testbed class. - - :param exp_name: The experiment name. - :param bzimage_path: Path of the bzimage. - :param initramfs_path: Path of the initramfs. - :param proj_name: Project name of the experiment. - :param password: Password of the user. - :param username: User of the VM. - :param public_key_path: path to the public key used to connect via - ssh to the virtual machines. - If `None`, password auth (root, root) - will be used. - :param use_vhost: Use virtual hosting or not? - - .. note:: In case no bzimage or initramfs is provided, Rumba - will automatically download the latest version available - from a repository. - """ - mod.Testbed.__init__(self, exp_name, - username, password, - proj_name, system_logs=['/var/log/messages']) - self.vms = {} - self.shims = [] - self.vhost = use_vhost - self.boot_processes = [] - self.bzimage_path = bzimage_path - self.initramfs_path = initramfs_path - self.multiproc_manager = None - self.key_path = public_key_path - - self.executor = SSHExecutor(self) - - # Prepend sudo to all commands if the user is not 'root' - def _may_sudo(self, cmds): - if os.geteuid() != 0: - for i, cmd in enumerate(cmds): - cmds[i] = "sudo %s" % cmd - - @staticmethod - def _run_command_chain(commands, results_queue, - error_queue, ignore_errors=False): - """ - Runs (sequentially) the command list. - - On error, breaks and dumps it in error_queue, and interrupts - as soon as it is non-empty (unless ignore errors is True). - - :type commands: list - :type results_queue: Queue - :type error_queue: Queue - :param commands: list of commands to execute - :param results_queue: Queue of results of parallel processes - :param error_queue: Queue of error(s) encountered - :return: None - """ - errors = 0 - # results_queue.cancel_join_thread() - # error_queue.cancel_join_thread() - for command in commands: - # if not error_queue.empty() and not ignore_errors: - # break - logger.debug('executing >> %s', command) - try: - subprocess.check_call(command.split()) - except (subprocess.CalledProcessError, IOError) as e: - error_queue.put(str(e)) - errors += 1 - if not ignore_errors: - break - except KeyboardInterrupt: - error_queue.put('Interrupted') - if errors == 0: - results_queue.put("Command chain ran correctly") - else: - results_queue.put("Command chain ran with %d error(s)" % errors) - - def _recover_if_names(self, experiment): - for node in experiment.nodes: - for ipcp in node.ipcps: - if isinstance(ipcp, mod.ShimEthIPCP): - shim_name, node_name = ipcp.name.split('.') - port_set = [x for x in self.vms[node_name]['ports'] - if x['shim'].name == shim_name] - port = port_set[0] - port_id = port['port_id'] - vm_id = self.vms[node_name]['id'] - mac = '00:0a:0a:0a:%02x:%02x' % (vm_id, port_id) - logger.info('Recovering ifname for port: %s.', - port['tap_id']) - output = node.execute_command('mac2ifname ' + mac) - ipcp.ifname = output - node.execute_command( - "ip link set dev %(ifname)s up" - % {'ifname': ipcp.ifname}) - - def _swap_in(self, experiment): - """ - :type experiment mod.Experiment - :param experiment: The experiment running - """ - - if os.geteuid() != 0: - try: - subprocess.check_call(["sudo", "-v"]) - if self.vhost and \ - (not os.access("/dev/vhost-net", os.R_OK) - or not os.access("/dev/vhost-net", os.W_OK) - or not os.access("/dev/kvm", os.R_OK) - or not os.access("/dev/kvm", os.W_OK)): - raise Exception('Cannot open vhost device. Make sure it is ' - 'available and you have rw permissions ' - 'on /dev/vhost-net') - except subprocess.CalledProcessError: - raise Exception('Not authenticated') - - logger.info("swapping in") - - # Download the proper buildroot images, if the user did not specify - # local images - url_prefix = "https://bitbucket.org/vmaffione/rina-images/downloads/" - if not self.bzimage_path: - bzimage = '%s.prod.bzImage' % (experiment.prototype_name()) - self.bzimage_path = os.path.join(mod.cache_dir, bzimage) - if not os.path.exists(self.bzimage_path): - logger.info("Downloading %s" % (url_prefix + bzimage)) - urlretrieve(url_prefix + bzimage, - filename=self.bzimage_path) - print("\n") - if not self.initramfs_path: - initramfs = '%s.prod.rootfs.cpio' % (experiment.prototype_name()) - self.initramfs_path = os.path.join(mod.cache_dir, initramfs) - if not os.path.exists(self.initramfs_path): - logger.info("Downloading %s" % (url_prefix + initramfs)) - urlretrieve(url_prefix + initramfs, - filename=self.initramfs_path) - print("\n") - - logger.info('Setting up interfaces.') - - # Building bridges and taps initialization scripts - br_tab_scripts = [] - for shim in experiment.dif_ordering: - if not isinstance(shim, mod.ShimEthDIF): - # Nothing to do here - continue - self.shims.append(shim) - ipcps = shim.ipcps - command_list = [] - command_list += ('brctl addbr %(br)s\n' - 'ip link set dev %(br)s up' - % {'br': shim.name} - ).split('\n') - for node in shim.members: # type:mod.Node - name = node.name - vm = self.vms.setdefault(name, {'vm': node, 'ports': []}) - port_id = len(vm['ports']) + 1 - tap_id = '%s.%02x' % (name, port_id) - - command_list += ('ip tuntap add mode tap name %(tap)s\n' - 'ip link set dev %(tap)s up\n' - 'brctl addif %(br)s %(tap)s' - % {'tap': tap_id, 'br': shim.name} - ).split('\n') - - # While we're at it, build vm ports table and ipcp table - vm['ports'].append({'tap_id': tap_id, - 'shim': shim, - 'port_id': port_id}) - ipcp_set = [x for x in ipcps if x in node.ipcps] - if len(ipcp_set) > 1: - raise Exception("Error: more than one ipcp in common " - "between shim dif %s and node %s" - % (shim.name, node.name)) - ipcp = ipcp_set[0] # type: mod.ShimEthIPCP - assert ipcp.name == '%s.%s' % (shim.name, node.name), \ - 'Incorrect Shim Ipcp found: expected %s.%s, found %s' \ - % (shim.name, node.name, ipcp.name) - ipcp.ifname = tap_id - # TODO deal with Ip address (shim UDP DIF). - - br_tab_scripts.append(command_list) - ## - # End of shim/node parsing block - ## - # - for node in experiment.nodes: - node.has_tcpdump = True - - def executor(list_of_commands): - for cmd in list_of_commands: - logger.debug('executing >> %s', cmd) - subprocess.check_call(cmd.split()) - - names = [] - args = [] - executors = [] - for i, script in enumerate(br_tab_scripts): - names.append(i) - self._may_sudo(script) - args.append(script) - executors.append(executor) - - m_processing.call_in_parallel(names, args, executors) - - logger.info('Interfaces setup complete. ' - 'Building VMs (this might take a while).') - - # Building vms - - boot_batch_size = max(1, old_m.cpu_count() // 2) - booting_budget = boot_batch_size - boot_backoff = 12 # in seconds - base_port = 2222 - vm_memory = 164 # in megabytes - vm_frontend = 'virtio-net-pci' - - vmid = 1 - - for node in experiment.nodes: - name = node.name - vm = self.vms.setdefault(name, {'vm': node, 'ports': []}) - vm['id'] = vmid - fwdp = base_port + vmid - fwdc = fwdp + 10000 - mac = '00:0a:0a:0a:%02x:%02x' % (vmid, 99) - vm['ssh'] = fwdp - vm['id'] = vmid - node.ssh_config.hostname = "localhost" - node.ssh_config.port = fwdp - node.ssh_config.username = self.username - node.ssh_config.password = self.password - log_file = os.path.join(mod.tmp_dir, name + '.log') - - vars_dict = {'fwdp': fwdp, 'id': vmid, 'mac': mac, - 'bzimage': self.bzimage_path, - 'initramfs': self.initramfs_path, - 'fwdc': fwdc, - 'memory': vm_memory, 'frontend': vm_frontend, - 'vmname': name, - 'log_file' : log_file} - - host_fwd_str = 'hostfwd=tcp::%(fwdp)s-:22' % vars_dict - vars_dict['hostfwdstr'] = host_fwd_str - - command = 'qemu-system-x86_64 ' - # TODO manage non default images - command += ('-kernel %(bzimage)s ' - '-append "console=ttyS0" ' - '-initrd %(initramfs)s ' - % vars_dict) - if os.path.exists('/dev/kvm'): - command += '--enable-kvm ' - command += ('-vga std ' - '-display none ' - '-smp 1 ' - '-m %(memory)sM ' - '-device %(frontend)s,mac=%(mac)s,netdev=mgmt ' - '-netdev user,id=mgmt,%(hostfwdstr)s ' - '-serial file:%(log_file)s ' - % vars_dict - ) - - del vars_dict - - for port in vm['ports']: - tap_id = port['tap_id'] - mac = '00:0a:0a:0a:%02x:%02x' % (vmid, port['port_id']) - port['mac'] = mac - - command += ( - '-device %(frontend)s,mac=%(mac)s,netdev=data%(idx)s ' - '-netdev tap,ifname=%(tap)s,id=data%(idx)s,script=no,' - 'downscript=no%(vhost)s ' - % {'mac': mac, 'tap': tap_id, 'idx': port['port_id'], - 'frontend': vm_frontend, - 'vhost': ',vhost=on' if self.vhost else ''} - ) - - booting_budget -= 1 - if booting_budget <= 0: - logger.debug('Sleeping %s secs waiting ' - 'for the VMs to boot', boot_backoff) - - time.sleep(boot_backoff) - booting_budget = boot_batch_size - - with open('%s/qemu_out_%s' % (mod.tmp_dir, vmid), 'w')\ - as out_file: - logger.debug('executing >> %s', command) - self.boot_processes.append(subprocess.Popen(command.split(), - stdout=out_file)) - - vmid += 1 - - # Wait for the last batch of VMs to start - if booting_budget < boot_batch_size: - tsleep = boot_backoff * (boot_batch_size - booting_budget) / \ - boot_batch_size - logger.debug('Sleeping %s secs ' - 'waiting for the last VMs to boot', - tsleep) - time.sleep(tsleep) - - logger.info('All VMs are running. Moving on...') - - self._recover_if_names(experiment) - - if self.key_path is not None: - for node in experiment.nodes: - try: - node.copy_file(self.key_path, '/root/.ssh') - node.execute_command( - 'cd /root/.ssh; ' - 'mv authorized_keys old.authorized_keys; ' - 'cat old.authorized_keys id_rsa.pub ' - '> authorized_keys' - ) - except SSHException as e: - logger.warning("Could not install ssh key into node %s. " - "%s.", node.name, str(e)) - logger.debug("Exception details:", exc_info=e) - - logger.info('Experiment has been successfully swapped in.') - - def _swap_out(self, experiment): - """ - :rtype str - :return: The script to tear down the experiment - """ - logger.info('Killing qemu processes.') - # TERM qemu processes - for process in self.boot_processes: - process.terminate() - - # Wait for them to shut down - for process in self.boot_processes: - process.wait() - - logger.info('Destroying interfaces.') - - names = [] - args = [] - executors = [] - - def executor(list_of_commands): - for cmd in list_of_commands: - logger.debug('executing >> %s', cmd) - try: - subprocess.check_call(cmd.split()) - except CalledProcessError as e: - logger.warning('Error during cleanup: %s', str(e)) - - index = 0 - for vm_name, vm in self.vms.items(): - for port in vm['ports']: - tap = port['tap_id'] - shim = port['shim'] - - commands = [] - commands += ('brctl delif %(br)s %(tap)s\n' - 'ip link set dev %(tap)s down\n' - 'ip tuntap del mode tap name %(tap)s' - % {'tap': tap, 'br': shim.name} - ).split('\n') - self._may_sudo(commands) - - names.append(index) - index += 1 - args.append(commands) - executors.append(executor) - - m_processing.call_in_parallel(names, args, executors) - - logger.info('Port tear-down complete. Destroying bridges.') - - names = [] - args = [] - executors = [] - - index = 0 - - for shim in self.shims: - commands = [] - commands += ('ip link set dev %(br)s down\n' - 'brctl delbr %(br)s' - % {'br': shim.name} - ).split('\n') - self._may_sudo(commands) - - names.append(index) - index += 1 - args.append(commands) - executors.append(executor) - - m_processing.call_in_parallel(names, args, executors) - - logger.info('Experiment has been swapped out.') diff --git a/rumba/topologies.py b/rumba/topologies.py new file mode 100644 index 0000000..de0e8fc --- /dev/null +++ b/rumba/topologies.py @@ -0,0 +1,146 @@ +# +# Topology builder helpers for rumba +# +# Copyright (C) 2017-2018 Nextworks S.r.l. +# Copyright (C) 2017-2018 imec +# +# Sander Vrijders <sander.vrijders@ugent.be> +# Dimitri Staessens <dimitri.staessens@ugent.be> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., http://www.fsf.org/about/contact/. +# + +"""Topology builder functions for common network shapes.""" + +from rumba.model import Node, UnicastLayer, EthDixLayer + + +def _node_name(names, i, prefix): + """Return the node name: from `names` list if given, else prefix + i.""" + if names is not None: + return names[i] + return prefix + str(i) + + +def build_chain(num_nodes, layer, prefix="node", shim_prefix="e", + names=None): + """Build a linear chain of nodes connected by EthDixLayers. + + Args: + num_nodes: Number of nodes in the chain. + layer: The UnicastLayer all nodes belong to. + prefix: Name prefix for nodes (default: "node"). + shim_prefix: Name prefix for EthDixLayers (default: "e"). + names: Optional list of node names (overrides prefix). + + Returns: + A list of Node objects forming a chain. + """ + nodes = [] + shim_prev = None + + for i in range(num_nodes): + shim = EthDixLayer(shim_prefix + str(i)) if i < num_nodes - 1 else None + + layers = [layer] + regs = [] + + if shim_prev is not None: + layers.append(shim_prev) + regs.append(shim_prev) + if shim is not None: + layers.append(shim) + regs.append(shim) + + node = Node(_node_name(names, i, prefix), layers=layers, + registrations={layer: regs}) + nodes.append(node) + shim_prev = shim + + return nodes + + +def build_ring(num_nodes, layer, prefix="node", shim_prefix="e", + names=None): + """Build a ring of nodes connected by EthDixLayers. + + Like build_chain, but the last shim links back to the first node. + + Args: + num_nodes: Number of nodes in the ring (must be >= 3). + layer: The UnicastLayer all nodes belong to. + prefix: Name prefix for nodes (default: "node"). + shim_prefix: Name prefix for EthDixLayers (default: "e"). + names: Optional list of node names (overrides prefix). + + Returns: + A list of Node objects forming a ring. + """ + nodes = [] + shim_prev = None + + for i in range(num_nodes): + shim = EthDixLayer(shim_prefix + str(i)) + + layers = [layer] + regs = [] + + if shim_prev is not None: + layers.append(shim_prev) + regs.append(shim_prev) + layers.append(shim) + regs.append(shim) + + node = Node(_node_name(names, i, prefix), layers=layers, + registrations={layer: regs}) + nodes.append(node) + shim_prev = shim + + # Close the ring: connect last shim back to first node + nodes[0].add_layer(shim_prev) + nodes[0].add_registration(layer, shim_prev) + + return nodes + + +def build_star(leaf_names, layer, hub_name="router", shim_prefix="e"): + """Build a star topology: leaf nodes connected to a central hub. + + Each leaf is connected to the hub via its own EthDixLayer. + + Args: + leaf_names: List of names for the leaf nodes. + layer: The UnicastLayer all nodes belong to. + hub_name: Name of the central hub node (default: "router"). + shim_prefix: Name prefix for EthDixLayers (default: "e"). + + Returns: + A tuple (leaves, hub) where leaves is a list of leaf Nodes + and hub is the central Node. + """ + leaves = [] + shims = [] + + for i, name in enumerate(leaf_names): + shim = EthDixLayer(shim_prefix + str(i + 1)) + shims.append(shim) + leaf = Node(name, layers=[shim, layer], + registrations={layer: [shim]}) + leaves.append(leaf) + + hub = Node(hub_name, layers=shims + [layer], + registrations={layer: shims}) + + return leaves, hub diff --git a/rumba/utils.py b/rumba/utils.py index 3a3b837..324d3bf 100644 --- a/rumba/utils.py +++ b/rumba/utils.py @@ -26,24 +26,19 @@ import enum import os +import time import rumba.log as log import rumba.model as model import rumba.testbeds.local as local # Fix input reordering -try: - import builtins # Only in Python 3 +import builtins - def input(prompt=''): - log.flush_log() - return builtins.input(prompt) -except ImportError: # We are in Python 2 - import __builtin__ - def input(prompt=''): - log.flush_log() - return __builtin__.raw_input(prompt) +def input(prompt=''): + log.flush_log() + return builtins.input(prompt) logger = log.get_logger(__name__) @@ -233,4 +228,64 @@ class ExperimentManager(object): # Make sure to print all logs before execution terminates, # Specifically the last two error logs above. return True - # Suppress the exception we logged: no traceback, unless logged. + + +# --------------------------------------------------------------------------- +# Polling utilities +# --------------------------------------------------------------------------- + +def wait_until(predicate, timeout=30, interval=0.5, description="condition"): + """ + Poll *predicate* until it returns a truthy value or *timeout* expires. + + :param predicate: A callable that returns True when the condition is met. + :param timeout: Maximum wait time in seconds. + :param interval: Polling interval in seconds. + :param description: Human-readable description (used in the error message). + :returns: The truthy value returned by *predicate*. + :raises TimeoutError: If *timeout* expires before *predicate* is truthy. + """ + deadline = time.monotonic() + timeout + last_exc = None + while time.monotonic() < deadline: + try: + result = predicate() + if result: + return result + except Exception as e: + last_exc = e + time.sleep(interval) + msg = "Timed out after %ss waiting for %s" % (timeout, description) + if last_exc is not None: + msg += " (last error: %s: %s)" % (type(last_exc).__name__, last_exc) + raise TimeoutError(msg) + + +def wait_for_command(node, command, timeout=30, interval=1.0, + description=None): + """ + Poll *command* on *node* until it succeeds (exit code 0). + + :param node: The node to run the command on. + :param command: Shell command string. + :param timeout: Maximum wait time in seconds. + :param interval: Polling interval in seconds. + :param description: Human-readable description for error messages. + :returns: A CommandResult on success. + :raises TimeoutError: If the command does not succeed within *timeout*. + """ + if description is None: + description = "'%s' to succeed" % command + + output = [None] + + def _try(): + try: + output[0] = node.execute_command(command, time_out=interval) + return True + except Exception: + return False + + wait_until(_try, timeout=timeout, interval=interval, + description=description) + return output[0] diff --git a/rumba/visualizer.py b/rumba/visualizer.py index a2f35cb..76660c0 100755..100644 --- a/rumba/visualizer.py +++ b/rumba/visualizer.py @@ -5,26 +5,46 @@ import plotly.graph_objs as go from math import sin, cos, pi, sqrt -__all__ = ['draw_network', 'draw_experiment', 'get_default_test_network'] +__all__ = ['draw_network', 'get_network_from_rumba_experiment', 'get_default_test_network'] import rumba.elements.topology +from rumba.elements.topology import LayerType type_to_marker = { + 'application': 'square-open', 'eth-dix': 'diamond', 'eth-llc': 'diamond', 'eth-udp': 'diamond', + 'local': 'diamond', 'unicast': 'circle', 'broadcast': 'square' } +# Map LayerType enum to visualizer type strings +_layer_type_to_viz = { + LayerType.ETH_DIX: 'eth-dix', + LayerType.ETH_LLC: 'eth-llc', + LayerType.UDP4: 'eth-udp', + LayerType.UDP6: 'eth-udp', + LayerType.LOCAL: 'local', + LayerType.UNICAST: 'unicast', + LayerType.BROADCAST: 'broadcast', +} + -def rumba_to_type(_type): - if _type == rumba.elements.topology.ShimEthIPCP: +def rumba_to_type(_type_or_layer): + """Convert a layer or IPCP class to a visualizer type string.""" + # Accept a Layer object directly + if hasattr(_type_or_layer, 'layer_type'): + return _layer_type_to_viz.get(_type_or_layer.layer_type, 'unicast') + # Accept IPCP class + if _type_or_layer == rumba.elements.topology.EthIPCP: return 'eth-dix' - if _type == rumba.elements.topology.ShimUDPIPCP: + if _type_or_layer == rumba.elements.topology.UdpIPCP: return 'eth-udp' - if _type == rumba.elements.topology.IPCP: + if _type_or_layer == rumba.elements.topology.IPCP: return 'unicast' + return 'unicast' def get_default_test_network(): @@ -66,11 +86,11 @@ def get_default_test_network(): } -def _get_nodes_in_dif(exp, dif): +def _get_nodes_in_layer(exp, layer): nodes = [] n = 0 for node in exp.nodes: - if dif in [d.name for d in node.difs]: + if layer in [l.name for l in node.layers]: nodes.append(n) n += 1 @@ -86,45 +106,46 @@ def _get_node_index(exp, node): return 0 -def _get_network_from_rumba_experiment(exp): +def get_network_from_rumba_experiment(exp): print(exp) print(exp.flows) _nw = dict() _nw['layers'] = dict() _nw['nodes'] = list(range(len(exp.nodes))) + _nw['nnames'] = [node.name for node in exp.nodes] _nw['registrations'] = dict() for node in exp.nodes: - for dif in node.difs: - if dif.name not in _nw['layers']: - _nw['layers'][dif.name] = dict() - _nw['layers'][dif.name]['type'] = rumba_to_type(dif.get_ipcp_class()) - _nw['layers'][dif.name]['nodes'] = _get_nodes_in_dif(exp, dif.name) - _nw['layers'][dif.name]['edges'] = list() - if _nw['layers'][dif.name]['type'] != 'unicast': # shim - nodes = _nw['layers'][dif.name]['nodes'] - _nw['layers'][dif.name]['edges'].append((nodes[0], nodes[1])) - _nw['registrations'][dif.name] = dict() + for layer in node.layers: + if layer.name not in _nw['layers']: + _nw['layers'][layer.name] = dict() + _nw['layers'][layer.name]['type'] = rumba_to_type(layer) + _nw['layers'][layer.name]['nodes'] = _get_nodes_in_layer(exp, layer.name) + _nw['layers'][layer.name]['edges'] = list() + if _nw['layers'][layer.name]['type'] != 'unicast': # shim + nodes = _nw['layers'][layer.name]['nodes'] + _nw['layers'][layer.name]['edges'].append((nodes[0], nodes[1])) + _nw['registrations'][layer.name] = dict() for layer in exp.flows: for flow in layer: if 'src' in flow and 'dst' in flow: src = _get_node_index(exp, flow['src'].node) dst = _get_node_index(exp, flow['dst'].node) - layer = flow['src'].dif.name + layer = flow['src'].layer.name _nw['layers'][layer]['edges'].append((src, dst)) src_regs = flow['src'].registrations dst_regs = flow['dst'].registrations - for dif in src_regs: - if dif.name not in _nw['registrations'][layer]: - _nw['registrations'][layer][dif.name] = set() - _nw['registrations'][layer][dif.name].add(src) - for dif in dst_regs: - if dif.name not in _nw['registrations'][layer]: - _nw['registrations'][layer][dif.name] = set() - _nw['registrations'][layer][dif.name].add(dst) + for reg in src_regs: + if reg.name not in _nw['registrations'][layer]: + _nw['registrations'][layer][reg.name] = set() + _nw['registrations'][layer][reg.name].add(src) + for reg in dst_regs: + if reg.name not in _nw['registrations'][layer]: + _nw['registrations'][layer][reg.name] = set() + _nw['registrations'][layer][reg.name].add(dst) return _nw @@ -226,7 +247,7 @@ def _create_system_coords(network): d = sqrt((x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1)) _min = d if _min is None else min(d, _min) - return _min + return _min if _min != 0 else 0.1 def _get_ipcps_for_system(network, system): @@ -297,11 +318,8 @@ def _create_ipcp_graph_data(network): _create_ipcp_network(network) _min = _create_system_coords(network) - print("_min = %s" % _min) _create_ipcp_coords(network, _min / 5) - print(network) - def _get_ipcp_attributes(network): coords = list() @@ -312,7 +330,7 @@ def _get_ipcp_attributes(network): for ipcp in network['_ipcps']: coords.append(ipcp['coords']) colors.append(ipcp['color']) - labels.append(ipcp['layer'] + ' ' + str(ipcp['node'])) + labels.append(ipcp['layer'] + ' ' + network['nnames'][ipcp['node']]) markers.append(type_to_marker[ipcp['type']]) return coords, colors, labels, markers @@ -463,12 +481,6 @@ def draw_network(network, name='Ouroboros network'): fig.show() - -def draw_experiment(exp, name='experiment'): - _nw = _get_network_from_rumba_experiment(exp) - draw_network(_nw, name) - - if __name__ == '__main__': nw = get_default_test_network() draw_network(nw) @@ -1,37 +1,53 @@ #!/usr/bin/env python +import re import setuptools -_locals = {} with open('rumba/_version.py') as fp: - exec(fp.read(), None, _locals) -version = _locals['__version__'] + _version_src = fp.read() +_match = re.search(r'__version_info__\s*=\s*\((\d+),\s*(\d+),\s*(\d+)\)', + _version_src) +version = '%s.%s.%s' % _match.groups() + +with open('README.md') as fp: + long_description = fp.read() setuptools.setup( name='Rumba', version=version, - url='https://gitlab.com/arcfire/rumba', - keywords='rina measurement testbed', - author='Sander Vrijders', - author_email='sander.vrijders@ugent.be', + url='https://codeberg.org/o7s/rumba', + keywords='ouroboros measurement testbed', + author='Sander Vrijders, Dimitri Staessens', + author_email='sander@ouroboros.rocks, dimitri@ouroboros.rocks', license='LGPL', - description='Rumba measurement framework for RINA', - packages=[ - 'rumba', - 'rumba.testbeds', - 'rumba.prototypes', - 'rumba.executors', - 'rumba.elements' + description='Rumba measurement framework for Ouroboros', + long_description=long_description, + long_description_content_type='text/markdown', + classifiers=[ + 'Development Status :: 4 - Beta', + 'License :: OSI Approved ' + ':: GNU Lesser General Public License v2 or later (LGPLv2+)', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', + 'Topic :: System :: Networking', ], + packages=setuptools.find_packages(exclude=['examples', 'tools', 'doc']), + python_requires='>=3.8', install_requires=[ - 'paramiko', - 'docker', - 'repoze.lru; python_version<"3.2"', - 'contextlib2; python_version<"3.0"', - 'enum34; python_version<"3.0"' + 'paramiko>=2.0,<4', + 'docker>=5.0,<8', ], - extras_require={'NumpyAcceleration': ['numpy'], - 'graphs': ['pydot'], - 'visualizer': ['igraph', 'plotly']}, - scripts=['tools/rumba-access'] + extras_require={ + 'NumpyAcceleration': ['numpy'], + 'graphs': ['pydot'], + 'visualizer': ['igraph', 'plotly'], + 'pyouroboros': ['PyOuroboros'], + }, + scripts=['tools/rumba-access'], ) diff --git a/tools/conf-examples/dc-vpns.conf b/tools/conf-examples/dc-vpns.conf deleted file mode 100644 index 8bfde51..0000000 --- a/tools/conf-examples/dc-vpns.conf +++ /dev/null @@ -1,114 +0,0 @@ -eth 110 100Mbps tor1 spine1 -eth 120 100Mbps tor1 spine2 -eth 11 25Mbps s11 tor1 -eth 12 25Mbps s12 tor1 -eth 13 25Mbps s13 tor1 -eth 14 25Mbps s14 tor1 -eth 15 25Mbps s15 tor1 -eth 16 25Mbps s16 tor1 -eth 17 25Mbps s17 tor1 -eth 18 25Mbps s18 tor1 -eth 210 100Mbps tor2 spine1 -eth 220 100Mbps tor2 spine2 -eth 21 25Mbps s21 tor2 -eth 22 25Mbps s22 tor2 -eth 23 25Mbps s23 tor2 -eth 24 25Mbps s24 tor2 -eth 25 25Mbps s25 tor2 -eth 26 25Mbps s26 tor2 -eth 27 25Mbps s27 tor2 -eth 28 25Mbps s28 tor2 -eth 310 100Mbps tor3 spine1 -eth 320 100Mbps tor3 spine2 -eth 31 25Mbps s31 tor3 -eth 32 25Mbps s32 tor3 -eth 33 25Mbps s33 tor3 -eth 34 25Mbps s34 tor3 -eth 35 25Mbps s35 tor3 -eth 36 25Mbps s36 tor3 -eth 37 25Mbps s37 tor3 -eth 38 25Mbps s38 tor3 -eth 410 100Mbps tor4 spine1 -eth 420 100Mbps tor4 spine2 -eth 41 25Mbps s41 tor4 -eth 42 25Mbps s42 tor4 -eth 43 25Mbps s43 tor4 -eth 44 25Mbps s44 tor4 -eth 45 25Mbps s45 tor4 -eth 46 25Mbps s46 tor4 -eth 47 25Mbps s47 tor4 -eth 48 25Mbps s48 tor4 - -# DIF dcfabric -dif dcfabric tor1 110 120 -dif dcfabric tor2 210 220 -dif dcfabric tor3 310 320 -dif dcfabric tor4 410 420 -dif dcfabric spine1 110 210 310 410 -dif dcfabric spine2 120 220 320 420 - -# DIF VPN1 -dif vpn1 s11 11 -dif vpn1 s12 12 -dif vpn1 s13 13 -dif vpn1 s14 14 -dif vpn1 tor1 11 12 13 14 dcfabric -dif vpn1 s21 21 -dif vpn1 s22 22 -dif vpn1 s23 23 -dif vpn1 s24 24 -dif vpn1 tor2 21 22 23 24 dcfabric - -# DIF VPN2 -dif vpn2 s31 31 -dif vpn2 s32 32 -dif vpn2 s33 33 -dif vpn2 s34 34 -dif vpn2 tor3 31 32 33 34 dcfabric -dif vpn2 s41 41 -dif vpn2 s42 42 -dif vpn2 s43 43 -dif vpn2 s44 44 -dif vpn2 tor4 41 42 43 44 dcfabric - -# DIF VPN3 -dif vpn3 s15 15 -dif vpn3 s16 16 -dif vpn3 s17 17 -dif vpn3 s18 18 -dif vpn3 tor1 15 16 17 18 dcfabric -dif vpn3 s25 25 -dif vpn3 s26 26 -dif vpn3 s27 27 -dif vpn3 s28 28 -dif vpn3 tor2 25 26 27 28 dcfabric - -# DIF VPN4 -dif vpn4 s35 35 -dif vpn4 s36 36 -dif vpn4 s37 37 -dif vpn4 s38 38 -dif vpn4 tor3 35 36 37 38 dcfabric -dif vpn4 s45 45 -dif vpn4 s46 46 -dif vpn4 s47 47 -dif vpn4 s48 48 -dif vpn4 tor4 45 46 47 48 dcfabric - -#Policies - -#Multipath FABRIC -#policy dcfabric spine1,spine2 rmt.pff multipath -#policy dcfabric spine1,spine2 routing link-state routingAlgorithm=ECMPDijkstra -#policy dcfabric * rmt cas-ps q_max=1000 -#policy dcfabric * efcp.*.dtcp cas-ps - -#Application to DIF mappings -#appmap vpn1 traffic.generator.server 1 -#appmap vpn1 rina.apps.echotime.server 1 -#appmap vpn2 traffic.generator.server 1 -#appmap vpn2 rina.apps.echotime.server 1 -#appmap vpn3 traffic.generator.server 1 -#appmap vpn3 rina.apps.echotime.server 1 -#appmap vpn4 traffic.generator.server 1 -#appmap vpn4 rina.apps.echotime.server 1 diff --git a/tools/conf-examples/geant2-renumber.conf b/tools/conf-examples/geant2-renumber.conf deleted file mode 100644 index 07c014c..0000000 --- a/tools/conf-examples/geant2-renumber.conf +++ /dev/null @@ -1,86 +0,0 @@ -eth 2000 100Mbps lisbon madrid -eth 2001 100Mbps lisbon london -eth 2002 100Mbps london dublin -eth 2003 100Mbps london paris -eth 2004 100Mbps london brussels -eth 2005 100Mbps paris madrid -eth 2006 100Mbps paris luxemburg -eth 2007 100Mbps paris bern -eth 2008 100Mbps madrid bern -eth 2009 100Mbps bern roma -eth 2010 100Mbps roma madrid -eth 2011 100Mbps brussels amsterdam -eth 2012 100Mbps roma valleta -eth 2013 100Mbps amsterdam valleta -eth 2014 100Mbps bern berlin -eth 2015 100Mbps luxemburg berlin -eth 2016 100Mbps amsterdam berlin -eth 2017 100Mbps amsterdam copenhagen -eth 2018 100Mbps berlin copenhagen -eth 2019 100Mbps copenhagen oslo -eth 2020 100Mbps oslo stockholm -eth 2021 100Mbps stockholm copenhagen -eth 2023 100Mbps copenhagen tallin -eth 2024 100Mbps tallin riga -eth 2025 100Mbps riga vilnius -eth 2026 100Mbps vilnius warsaw -eth 2027 100Mbps warsaw berlin -eth 2028 100Mbps warsaw praha -eth 2029 100Mbps berlin praha -eth 2030 100Mbps berlin viena -eth 2031 100Mbps praha viena -eth 2032 100Mbps viena budapest -eth 2034 100Mbps viena ljubljana -eth 2035 100Mbps ljubljana zagreb -eth 2036 100Mbps zagreb budapest -eth 2037 100Mbps budapest sofia -eth 2038 100Mbps viena athens -eth 2039 100Mbps sofia athens -eth 2040 100Mbps athens roma -eth 2041 100Mbps sofia bucharest -eth 2042 100Mbps bucharest budapest -eth 2043 100Mbps athens nicosia -eth 2044 100Mbps roma nicosia -eth 2045 100Mbps sofia ankara -eth 2046 100Mbps bucharest ankara -eth 2047 100Mbps berlin moscow -eth 2048 100Mbps copenhagen moscow -eth 2049 100Mbps roma viena - -# DIF renumber -dif renumber lisbon 2000 2001 -dif renumber madrid 2000 2005 -dif renumber london 2001 2002 2003 2004 -dif renumber dublin 2002 -dif renumber paris 2003 2005 2006 2007 -dif renumber brussels 2004 2011 -dif renumber luxemburg 2006 2015 -dif renumber bern 2007 2008 2009 2014 -dif renumber roma 2009 2010 2012 2040 2044 2049 -dif renumber amsterdam 2011 2013 2016 2017 -dif renumber valleta 2012 2013 -dif renumber berlin 2014 2015 2016 2018 2027 2029 2030 2047 -dif renumber copenhagen 2017 2018 2019 2021 2023 2048 -dif renumber oslo 2019 2020 -dif renumber stockholm 2020 2021 -dif renumber tallin 2023 2024 -dif renumber riga 2024 2025 -dif renumber vilnius 2025 2026 -dif renumber warsaw 2026 2027 2028 -dif renumber praha 2028 2029 2031 -dif renumber viena 2030 2031 2032 2034 2038 -dif renumber budapest 2032 2036 2037 2042 -dif renumber athens 2038 2039 2040 2043 -dif renumber ljubljana 2034 2035 -dif renumber zagreb 2035 2036 -dif renumber sofia 2037 2039 2041 2045 -dif renumber bucharest 2041 2042 2046 -dif renumber nicosia 2043 2044 -dif renumber ankara 2045 2046 -dif renumber moscow 2047 2048 - -#Policies - -#address-change -policy renumber * namespace-manager address-change useNewTimeout=20001 deprecateOldTimeout=80001 changePeriod=120001 addressRange=100 -policy renumber * routing link-state objectMaximumAge=10000 waitUntilReadCDAP=5001 waitUntilError=5001 waitUntilPDUFTComputation=103 waitUntilFSODBPropagation=101 waitUntilAgeIncrement=997 waitUntilDeprecateAddress=20001 routingAlgorithm=Dijkstra diff --git a/tools/conf-examples/insane-stacking.conf b/tools/conf-examples/insane-stacking.conf deleted file mode 100644 index 8032fea..0000000 --- a/tools/conf-examples/insane-stacking.conf +++ /dev/null @@ -1,29 +0,0 @@ -eth 300 0Mbps a b - -# DIF n1 lays over shim DIF 300 -dif n1 a 300 -dif n1 b 300 - -# n2 lays over n1 -dif n2 a n1 -dif n2 b n1 - -# n3 lays over n2 -dif n3 a n2 -dif n3 b n2 - -# n4 lays over n3 -dif n4 a n3 -dif n4 b n3 - -# n5 lays over n4 -dif n5 a n4 -dif n5 b n4 - -# n6 lays over n5 -dif n6 a n5 -dif n6 b n5 - -# n7 lays over n6 -dif n7 a n6 -dif n7 b n6 diff --git a/tools/conf-examples/isp-sec.conf b/tools/conf-examples/isp-sec.conf deleted file mode 100644 index 33a35a6..0000000 --- a/tools/conf-examples/isp-sec.conf +++ /dev/null @@ -1,189 +0,0 @@ -eth 110 0Mbps cpe11 ar1 -eth 120 0Mbps cpe12 ar1 -eth 130 0Mbps cpe13 ar1 -eth 210 0Mbps cpe21 ar2 -eth 220 0Mbps cpe22 ar2 -eth 230 0Mbps cpe23 ar2 -eth 310 0Mbps cpe31 ar3 -eth 320 0Mbps cpe32 ar3 -eth 330 0Mbps cpe33 ar3 -eth 100 0Mbps ar1 manpe1 -eth 200 0Mbps ar2 manpe1 -eth 300 0Mbps ar3 manpe2 -eth 410 0Mbps manpe1 manpe2 -eth 411 0Mbps manpe1 manpe3 -eth 412 0Mbps manpe1 manpe4 -eth 420 0Mbps manpe2 manpe3 -eth 421 0Mbps manpe2 manpe4 -eth 430 0Mbps manpe3 manpe4 -eth 510 0Mbps manpe3 ser1 -eth 520 0Mbps manpe4 ser2 -eth 600 0Mbps ser1 core1 -eth 610 0Mbps ser1 core2 -eth 620 0Mbps ser2 core1 -eth 630 0Mbps ser2 core2 -eth 700 0Mbps core1 core2 -eth 710 0Mbps core1 core3 -eth 720 0Mbps core2 core4 -eth 730 0Mbps core3 core4 -eth 640 0Mbps core3 edge1 -eth 650 0Mbps core4 edge1 -eth 660 0Mbps core3 edge2 -eth 670 0Mbps core4 edge2 -eth 800 0Mbps edge1 isp2 -eth 810 0Mbps edge1 isp3 -eth 820 0Mbps edge2 isp4 -eth 830 0Mbps edge2 isp5 - -# DIF core -dif core ser1 600 610 -dif core ser2 620 630 -dif core core1 600 620 700 710 -dif core core2 610 630 700 720 -dif core core3 640 660 710 730 -dif core core4 650 670 720 730 -dif core edge1 640 650 -dif core edge2 660 670 - -# DIF access -dif access ar1 100 -dif access ar2 200 -dif access ar3 300 -dif access manpe1 100 200 410 411 412 -dif access manpe2 300 410 420 421 -dif access manpe3 411 420 430 510 -dif access manpe4 412 421 430 520 -dif access ser1 510 -dif access ser2 520 - -# DIF service -dif service ar1 access -dif service ar2 access -dif service ar3 access -dif service ser1 access core -dif service ser2 access core -dif service edge1 core -dif service edge2 core - -# DIF emall1 -dif emall1 cpe11 110 -dif emall1 cpe12 120 -dif emall1 cpe21 210 -dif emall1 cpe22 220 -dif emall1 cpe31 310 -dif emall1 ar1 110 120 service -dif emall1 ar2 210 220 service -dif emall1 ar3 310 service -dif emall1 edge1 service 800 -dif emall1 edge2 service 820 -dif emall1 isp2 800 -dif emall1 isp4 820 - -# DIF emall2 -dif emall2 cpe13 130 -dif emall2 cpe23 230 -dif emall2 cpe32 320 -dif emall2 cpe33 330 -dif emall2 ar1 130 service -dif emall2 ar2 230 service -dif emall2 ar3 320 330 service -dif emall2 edge1 service 810 -dif emall2 edge2 service 830 -dif emall2 isp3 810 -dif emall2 isp5 830 - -#policies -policy emall1 * security-manager.auth.default PSOC_authentication-ssh2 keyExchangeAlg=EDH keystore=/creds/ssh2 keystorePass=test -policy emall1 * security-manager.encrypt.default default encryptAlg=AES128 macAlg=SHA256 compressAlg=deflate -policy emall1 ar1,ar2,ar3,edge1,edge2 security-manager.auth.service PSOC_authentication-none -policy emall2 * security-manager.auth.default PSOC_authentication-ssh2 keyExchangeAlg=EDH keystore=/creds/ssh2 keystorePass=test -policy emall2 * security-manager.encrypt.default default encryptAlg=AES128 macAlg=SHA256 compressAlg=deflate -policy emall2 ar1,ar2,ar3,edge1,edge2 security-manager.auth.service PSOC_authentication-none - -#Enrollments -enroll access ar1 manpe1 100 -enroll access ar2 manpe1 200 -enroll access ar3 manpe2 300 -enroll access ser1 manpe3 510 -enroll access ser2 manpe4 520 -enroll access manpe1 manpe2 410 -enroll access manpe1 manpe3 411 -enroll access manpe1 manpe4 412 -enroll access manpe2 manpe3 420 -enroll access manpe2 manpe4 421 -enroll access manpe3 manpe4 430 - -enroll core core1 core2 700 -enroll core core1 core3 710 -enroll core core2 core4 720 -enroll core core3 core4 730 -enroll core ser1 core1 600 -enroll core ser1 core2 610 -enroll core ser2 core1 620 -enroll core ser2 core2 630 -enroll core edge1 core3 640 -enroll core edge1 core4 650 -enroll core edge2 core3 660 -enroll core edge2 core4 670 - -enroll service edge1 edge2 core -enroll service edge1 ser1 core -enroll service edge1 ser2 core -enroll service edge2 ser1 core -enroll service edge2 ser2 core -enroll service ser1 ser2 core -enroll service ar1 ser1 access -enroll service ar1 ser2 access -enroll service ar2 ser1 access -enroll service ar2 ser2 access -enroll service ar3 ser1 access -enroll service ar3 ser2 access - -enroll emall1 cpe11 ar1 110 -enroll emall1 cpe12 ar1 120 -enroll emall1 cpe21 ar2 210 -enroll emall1 cpe22 ar2 220 -enroll emall1 cpe31 ar3 310 -enroll emall1 ar1 edge1 service -enroll emall1 ar1 edge2 service -enroll emall1 ar2 edge1 service -enroll emall1 ar2 edge2 service -enroll emall1 ar3 edge1 service -enroll emall1 ar3 edge2 service -enroll emall1 edge1 edge2 service -enroll emall1 isp2 edge1 800 -enroll emall1 isp4 edge2 820 - -enroll emall2 cpe13 ar1 130 -enroll emall2 cpe23 ar2 230 -enroll emall2 cpe32 ar3 320 -enroll emall2 cpe33 ar3 330 -enroll emall2 ar1 edge1 service -enroll emall2 ar1 edge2 service -enroll emall2 ar2 edge1 service -enroll emall2 ar2 edge2 service -enroll emall2 ar3 edge1 service -enroll emall2 ar3 edge2 service -enroll emall2 edge1 edge2 service -enroll emall2 isp3 edge1 810 -enroll emall2 isp5 edge2 830 - -#Overlays -overlay ar1 overlays/ispsec/ar1 -overlay ar2 overlays/ispsec/ar2 -overlay ar3 overlays/ispsec/ar3 -overlay cpe11 overlays/ispsec/cpe11 -overlay cpe12 overlays/ispsec/cpe12 -overlay cpe13 overlays/ispsec/cpe13 -overlay cpe21 overlays/ispsec/cpe21 -overlay cpe22 overlays/ispsec/cpe22 -overlay cpe23 overlays/ispsec/cpe23 -overlay cpe31 overlays/ispsec/cpe31 -overlay cpe32 overlays/ispsec/cpe32 -overlay cpe33 overlays/ispsec/cpe33 -overlay edge1 overlays/ispsec/edge1 -overlay edge2 overlays/ispsec/edge2 -overlay isp2 overlays/ispsec/isp2 -overlay isp3 overlays/ispsec/isp3 -overlay isp4 overlays/ispsec/isp4 -overlay isp5 overlays/ispsec/isp5 diff --git a/tools/conf-examples/resilient-square.conf b/tools/conf-examples/resilient-square.conf deleted file mode 100644 index 592b6a5..0000000 --- a/tools/conf-examples/resilient-square.conf +++ /dev/null @@ -1,16 +0,0 @@ -# a, b and c and d are connected through p2p shim DIFs, in circle. -# Between a and c there is an additional diagonal link. -eth 300 100Mbps a b -eth 400 100Mbps b c -eth 500 100Mbps c d -eth 600 1Mbps d a -eth 700 100Mbps a c - -# DIF n1 spans over the p2p shim DIFs -dif n1 a 300 600 700 -dif n1 b 300 400 -dif n1 c 400 500 700 -dif n1 d 500 600 - -# Use LFA policy as PDU Forwarding Function -policy n1 * rmt.pff lfa diff --git a/tools/conf-examples/secure-two-layers.conf b/tools/conf-examples/secure-two-layers.conf deleted file mode 100644 index 54c1da6..0000000 --- a/tools/conf-examples/secure-two-layers.conf +++ /dev/null @@ -1,25 +0,0 @@ -eth 300 0Mbps a b -eth 400 0Mbps b c -eth 500 0Mbps c d - -# DIF n1 spans a,b and c and runs over the shims -dif n1 a 300 -dif n1 b 300 400 -dif n1 c 400 - -# DIF n2 spans c and d and runs over the shims -dif n2 c 500 -dif n2 d 500 - -# DIF n3 spans over n1 and n2 -dif n3 a n1 -dif n3 c n1 n2 -dif n3 d n2 - -policy n3 * security-manager.auth.default PSOC_authentication-ssh2 keyExchangeAlg=EDH keystore=/creds keystorePass=test -policy n3 * security-manager.encrypt.default default encryptAlg=AES128 macAlg=SHA256 compressAlg=deflate -policy n3 * security-manager.ttl.default default initialValue=50 -policy n3 * security-manager.errorcheck.default CRC32 -policy n3 * security-manager.auth.n1 PSOC_authentication-password password=kf05j.a1234.af0k -policy n3 * security-manager.ttl.n1 default initialValue=50 -policy n3 * security-manager.errorcheck.n1 CRC32 diff --git a/tools/conf-examples/seven.conf b/tools/conf-examples/seven.conf deleted file mode 100644 index b25f476..0000000 --- a/tools/conf-examples/seven.conf +++ /dev/null @@ -1,34 +0,0 @@ -# This configuration realizes the following seven-nodes topology -# -# MA ---- MB ---- MC --- MD --- ME -# | | -# MF MG -# - -# 300 is a shim-eth-vlan DIF, with nodes a and b -eth 300 0Mbps a b - -# 400 is a shim-eth-vlan DIF, with nodes b and c -eth 400 0Mbps b c - -# 500 is a shim-eth-vlan DIF, with nodes c and f -eth 500 0Mbps c f - -# 600 is a shim-eth-vlan DIF, with nodes c and d -eth 600 0Mbps c d - -# 700 is a shim-eth-vlan DIF, with nodes d and e -eth 700 0Mbps d e - -# 800 is a shim-eth-vlan DIF, with nodes e and g -eth 800 0Mbps e g - -# DIF n1 spans over the two shim DIFs -dif n1 a 300 -dif n1 b 300 400 -dif n1 c 400 500 600 -dif n1 d 600 700 -dif n1 e 700 800 -dif n1 f 500 -dif n1 g 800 - diff --git a/tools/conf-examples/star.conf b/tools/conf-examples/star.conf deleted file mode 100644 index 8a4f6ab..0000000 --- a/tools/conf-examples/star.conf +++ /dev/null @@ -1,7 +0,0 @@ -# a,b and c are in the same L2 domain -eth 300 0Mbps a b c - -# DIF n1 spans over the shim DIF -dif n1 a 300 -dif n1 b 300 -dif n1 c 300 diff --git a/tools/conf-examples/triangle.conf b/tools/conf-examples/triangle.conf deleted file mode 100644 index f89811c..0000000 --- a/tools/conf-examples/triangle.conf +++ /dev/null @@ -1,9 +0,0 @@ -# a, b and c are connected through p2p shim DIFs -eth 300 10Mbps a b -eth 400 20Mbps b c -eth 500 30Mbps a c - -# DIF n1 spans over the p2p shim DIFs -dif n1 a 300 500 -dif n1 b 300 400 -dif n1 c 400 500 diff --git a/tools/conf-examples/tutorial1.conf b/tools/conf-examples/tutorial1.conf deleted file mode 100644 index 8023687..0000000 --- a/tools/conf-examples/tutorial1.conf +++ /dev/null @@ -1,4 +0,0 @@ -eth 100 0Mbps system1 system2 - -dif Normal system1 100 -dif Normal system2 100 diff --git a/tools/conf-examples/tutorial2.conf b/tools/conf-examples/tutorial2.conf deleted file mode 100644 index b43fc17..0000000 --- a/tools/conf-examples/tutorial2.conf +++ /dev/null @@ -1,6 +0,0 @@ -eth 100 0Mbps system1 system2 -eth 101 0Mbps system2 system3 - -dif Normal system1 100 -dif Normal system2 100 101 -dif Normal system3 101 diff --git a/tools/conf-examples/two-layers.conf b/tools/conf-examples/two-layers.conf deleted file mode 100644 index dc1bab2..0000000 --- a/tools/conf-examples/two-layers.conf +++ /dev/null @@ -1,17 +0,0 @@ -eth 300 0Mbps a b -eth 400 0Mbps b c -eth 500 0Mbps c d - -# DIF n1 spans a,b and c and runs over the shims -dif n1 a 300 -dif n1 b 300 400 -dif n1 c 400 - -# DIF n2 spans c and d and runs over the shims -dif n2 c 500 -dif n2 d 500 - -# DIF n3 spans over n1 and n2 -dif n3 a n1 -dif n3 c n1 n2 -dif n3 d n2 diff --git a/tools/democonf2rumba.py b/tools/democonf2rumba.py deleted file mode 100755 index 795cf22..0000000 --- a/tools/democonf2rumba.py +++ /dev/null @@ -1,292 +0,0 @@ -#!/usr/bin/env python - -import argparse -import re - -import rumba.model as mod -import rumba.log as log - - -def make_experiment(filename, experiment_class, experiment_kwargs, - testbed_class, testbed_kwargs, verbosity): - """ - :type filename str - :param filename: path to the .conf file - :param experiment_class: subclass of mod.Experiment - :param experiment_kwargs: args dict for experiment constructor - (nodes will be overwritten) - :param testbed_class: subclass of mod.Testbed - :param testbed_kwargs: args dict for experiment constructor - (nodes will be overwritten) - """ - shims = {} - nodes = {} - difs = {} - - print('Reading file %s.' % (filename,)) - print('+++++++++++++++++++') - print() - - with open(filename, 'r') as conf: - - line_cnt = 0 - - while 1: - line = conf.readline() - if line == '': - break - line_cnt += 1 - - line = line.replace('\n', '').strip() - - if line.startswith('#') or line == "": - continue - - m = re.match(r'\s*eth\s+([\w-]+)\s+(\d+)([GMK])bps\s+(\w.*)$', line) - if m: - shim = m.group(1) - speed = int(m.group(2)) - speed_unit = m.group(3).lower() - vm_list = m.group(4).split() - - if shim in shims or shim in difs: - print('Error: Line %d: shim %s already defined' - % (line_cnt, shim)) - continue - - if speed_unit == 'K': - speed = speed // 1000 - if speed_unit == 'G': - speed = speed * 1000 - - shims[shim] = {'name': shim, - 'speed': speed, - 'type': 'eth'} - - for vm in vm_list: - nodes.setdefault(vm, {'name': vm, 'difs': [], - 'dif_registrations': {}, - 'registrations': {}}) - nodes[vm]['difs'].append(shim) - continue - - m = re.match(r'\s*dif\s+([\w-]+)\s+([\w-]+)\s+(\w.*)$', line) - if m: - dif = m.group(1) - vm = m.group(2) - dif_list = m.group(3).split() - - if dif in shims: - print('Error: Line %d: dif %s already defined as shim' - % (line_cnt, dif)) - continue - - difs.setdefault(dif, { - 'name': dif}) # Other dict contents might be policies. - - if vm in nodes and dif in nodes[vm]['dif_registrations']: - print('Error: Line %d: vm %s in dif %s already specified' - % (line_cnt, vm, dif)) - continue - - nodes.setdefault(vm, {'name': vm, 'difs': [], - 'dif_registrations': {}, - 'registrations': {}}) - nodes[vm]['difs'].append(dif) - nodes[vm]['dif_registrations'][dif] = dif_list - # It is not defined yet, per check above. - - continue - - # No match, spit a warning - print('Warning: Line %d unrecognized and ignored' % line_cnt) - - # File parsed - - parsed_difs = {} - - for shim_name, shim in shims.items(): - parsed_difs[shim_name] = (mod.ShimEthDIF( - shim_name, - link_quality=mod.LinkQuality(rate=shim['speed'])) - ) - - for dif_name, dif in difs.items(): - parsed_difs[dif_name] = (mod.NormalDIF(dif_name)) - - parsed_nodes = [] - for node, node_data in nodes.items(): - name = node_data['name'] - difs = [parsed_difs[x] for x in node_data['difs']] - dif_registrations = {parsed_difs[x]: [parsed_difs[y] for y in l] - for x, l in node_data['dif_registrations'] - .items()} - parsed_nodes.append(mod.Node(name, difs, dif_registrations)) - - log.set_logging_level(verbosity) - print() - print('++++++++++++++++++++') - print('Calling constructor of testbed %s with args %s.' - % (testbed_class, testbed_kwargs)) - print('++++++++++++++++++++') - print() - - testbed = testbed_class(**testbed_kwargs) - - experiment_kwargs['testbed'] = testbed - experiment_kwargs['nodes'] = parsed_nodes - - exp = experiment_class(**experiment_kwargs) - try: - exp.swap_in() - exp.bootstrap_prototype() - input("Press ENTER to quit") - finally: - exp.swap_out() - - -def setup_testbed_common_args(t_p): - - t_p.add_argument('-E', '--exp_name', metavar='EXP_NAME', type=str, - required=True, - help='Experiment name') - t_p.add_argument('-U', '--username', metavar='USERNAME', type=str, - required=True, - help='Testbed user name') - t_p.add_argument('-P', '--proj_name', metavar='PROJECT_NAME', type=str, - help='Project name') - t_p.add_argument('-W', '--password', metavar='PASSWORD', type=str, - help='Testbed password') - - -if __name__ == '__main__': - description = "Demonstrator config file to rumba script converter" - epilog = "2017 Marco Capitani <m.capitani@nextworks.it>" - - parser = argparse.ArgumentParser(description=description, - epilog=epilog) - - parser.add_argument('-P', '--prototype', type=str, required=True, - choices=['irati', 'ouroboros', 'rlite'], - help='The kind of prototype plugin to use to run' - ' the experiment.') - - parser.add_argument('-C', '--conf', metavar='CONFIG', type=str, - required=True, - help='Path to the config file to parse') - - parser.add_argument( - '--verbosity', metavar='VERBOSITY', type=str, - default='INFO', - choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], - help='Rumba output verbosity') - - subparsers = parser.add_subparsers(dest='testbed') - emulab_p = subparsers.add_parser('emulab', help='Use emulab testbed') - jfed_p = subparsers.add_parser('jfed', help='Use jfed testbed') - qemu_p = subparsers.add_parser('qemu', help='Use qemu testbed') - fake_p = subparsers.add_parser('fake', help='Use fake testbed') - - for t in [emulab_p, jfed_p, qemu_p, fake_p]: - setup_testbed_common_args(t) - - qemu_p.add_argument('-B', '--bzimage-path', metavar='BZIMAGE', type=str, - required=True, - help='path to the bzImage file to use') - qemu_p.add_argument('-I', '--initramfs-path', metavar='INITRAMFS', type=str, - required=True, - help='path to the initramfs file to use') - qemu_p.add_argument('-V', '--use_vhost', action='store_true', - default=False, help='Use vhost') - qemu_p.add_argument('-Q', '--qemu_logs_dir', metavar='QEMU_LOGS', type=str, - default=None, help='path to the folder for qemu logs') - qemu_p.add_argument('--public-key-path', metavar='PATH', type=str, - default=None, help='path to the user ssh public key.') - - emulab_p.add_argument('-R', '--url', metavar='URL', type=str, - default="wall2.ilabt.iminds.be", - help='Url') - emulab_p.add_argument('-I', '--image', metavar='IMG', type=str, - default="UBUNTU14-64-STD", - help='Ubuntu image') - - jfed_p.add_argument('-C', '--cert_file', metavar='CERT', type=str, - required=True, - help='Certificate file') - jfed_p.add_argument('-H', '--exp_hours', metavar='HOURS', type=str, - default="2", help='Experiment hours') - jfed_p.add_argument('-A', '--authority', metavar='AUTH', type=str, - default="wall2.ilabt.iminds.be", - help='Authority') - jfed_p.add_argument('-I', '--image', metavar='IMAGE', type=str, - default=None, - help='Image to be used') - jfed_p.add_argument('--image-custom', metavar='I_CUSTOM', type=bool, - default=False, - help='Is the image a custom one?') - jfed_p.add_argument('--image-owner', metavar='I_OWNER', type=str, - default=None, - help='Creator of the image') - jfed_p.add_argument('--use-physical-machines', metavar='USE_PM', type=bool, - default=None, - help='Try to allocate physical machines if True') - - args = parser.parse_args() - - if args.testbed == 'emulab': - import rumba.testbeds.emulab as emulab - test_class = emulab.Testbed - testbed_args = {a.dest: getattr(args, a.dest) - for a in emulab_p._actions - if a.dest != 'help' - and getattr(args, a.dest) is not None} - elif args.testbed == 'jfed': - import rumba.testbeds.jfed as jfed - test_class = jfed.Testbed - testbed_args = {a.dest: getattr(args, a.dest) - for a in jfed_p._actions - if a.dest != 'help' - and getattr(args, a.dest) is not None} - elif args.testbed == 'qemu': - import rumba.testbeds.qemu as qemu - test_class = qemu.Testbed - testbed_args = {a.dest: getattr(args, a.dest) - for a in qemu_p._actions - if a.dest != 'help' - and getattr(args, a.dest) is not None} - elif args.testbed == 'local': - import rumba.testbeds.local as local - test_class = local.Testbed - testbed_args = {a.dest: getattr(args, a.dest) - for a in fake_p._actions - if a.dest != 'help' - and getattr(args, a.dest) is not None} - else: - if args.testbed is None: - print('Testbed type must be specified!') - print(parser.format_help()) - exit(1) - raise ValueError('Unexpected testbed: %s.' % args.testbed) - - if args.prototype == 'irati': - import rumba.prototypes.irati as irati - exp_class = irati.Experiment - elif args.prototype == 'ouroboros': - import rumba.prototypes.ouroboros as ouroboros - exp_class = ouroboros.Experiment - elif args.prototype == 'rlite': - import rumba.prototypes.rlite as rlite - exp_class = rlite.Experiment - else: - raise ValueError('Unexpected prototype: %s.' % args.testbed) - - try: - make_experiment(args.conf, - experiment_class=exp_class, - experiment_kwargs={}, - testbed_class=test_class, - testbed_kwargs=testbed_args, - verbosity=args.verbosity) - - except KeyboardInterrupt: - print("Interrupted. Closing down.") diff --git a/tools/scriptgenerator.py b/tools/scriptgenerator.py index cc3e1ea..f986185 100644 --- a/tools/scriptgenerator.py +++ b/tools/scriptgenerator.py @@ -6,22 +6,22 @@ import rumba.log as log from rumba.utils import ExperimentManager, PAUSE_SWAPOUT client1 = Client( - "rinaperf", - options="-t perf -s 1000 -c 0", - c_id='rinaperf_^C' # To differentiate + "operf", + options="-s 1000 -d 0", + c_id='operf_flood' # To differentiate ) client2 = Client( - "rinaperf", - options="-t perf -s 1000 -D <duration>", + "operf", + options="-s 1000 -d <duration>", shutdown="", - c_id='rinaperf_-D' # To differentiate + c_id='operf_timed' # To differentiate ) server = Server( - "rinaperf", + "operf", options="-l", arrival_rate=0.5, mean_duration=5, |