storyboard: add logging and log retrieval

1 files changed, 47 insertions, 10 deletions

diff --git a/rumba/storyboard.py b/rumba/storyboard.py
index 52bd875..4a09ac8 100644
--- a/rumba/storyboard.py
+++ b/rumba/storyboard.py
@@ -5,6 +5,7 @@
 #    Copyright (C) 2017 imec
 #
 #    Sander Vrijders   <sander.vrijders@ugent.be>
+#    Dimitri Staessens <dimitri.staessens@ugent.be>
 #    Vincenzo Maffione <v.maffione@nextworks.it>
 #    Marco Capitani    <m.capitani@nextworks.it>
 #
@@ -23,11 +24,12 @@
 # Foundation, Inc., http://www.fsf.org/about/contact/.
 #
 
-# Base class for client programs
+# Base class for client apps
 #
 # @ap: Application Process binary
 # @options: Options to pass to the binary
 #
+import os
 import random
 import time
 
@@ -50,6 +52,14 @@ except ImportError:
     logger.debug("Falling back to simple implementations.")
     # PROBLEM! These logs will almost never be printed... But we might not care
 
+current_id = -1
+
+
+def get_id():
+    global current_id
+    current_id += 1
+    return current_id
+
 
 class Client(object):
     def __init__(self, ap, nodes=None, options=None, shutdown="kill <pid>"):
@@ -69,7 +79,7 @@ class Client(object):
 
     def process(self, duration):
         node = random.choice(self.nodes) if len(self.nodes) > 0 else None
-        return ClientProcess(self.ap, self.startup, duration, node, self.shutdown)
+        return ClientProcess(get_id(), self.ap, self.startup, duration, node, self.shutdown)
 
 
 # Base class for client processes
@@ -80,8 +90,9 @@ class Client(object):
 # @options: Options to pass to the binary
 #
 class ClientProcess(object):
-    def __init__(self, ap, startup, duration,
+    def __init__(self, client_id, ap, startup, duration,
                  node=None, shutdown="<kill <pid>"):
+        self.id = client_id
         self.ap = ap
         self.startup = startup
         self.duration = duration
@@ -103,7 +114,9 @@ class ClientProcess(object):
             self.ap, self.node.name, self.duration
         )
 
-        start_cmd = "./startup.sh %s" % (
+        start_cmd = "./startup.sh %s_%s %s" % (
+            self.ap,
+            self.id,
             self.startup.replace("<duration>", str(self.duration)),
         )
         self.running = True
@@ -209,11 +222,11 @@ class Server:
     def run(self):
         for node in self.nodes:
             logfile = "%s_server.log" % self.ap
-            script = r'nohup "$@" > %s & echo "$!"' % (logfile,)
+            script = r'nohup "$@" > %s 2>&1 & echo "$!"' % (logfile,)
             run_cmd = self.ap + (
                 (" " + self.options) if self.options is not None else ""
             )
-            cmd_1 = "echo '%s' > startup.sh && chmod a+x startup.sh && cat startup.sh" \
+            cmd_1 = "echo '%s' > startup.sh && chmod a+x startup.sh" \
                     % (script,)
             cmd_2 = "./startup.sh %s" % (run_cmd,)
             logger.debug(
@@ -223,7 +236,6 @@ class Server:
             try:
                 node.execute_command(cmd_1)
                 self.pids[node] = (node.execute_command(cmd_2))
-                node.execute_command("cat startup.sh")
             except ssh_support.SSHException:
                 logger.warn('Could not start server %s on node %s.',
                             self.ap, node.name)
@@ -269,7 +281,8 @@ class StoryBoard:
                              "setting the experiment.")
         if hasattr(s, '__len__') and len(s) == 2:
             server, node = s
-            if not isinstance(server, Server) or not isinstance(node, model.Node):
+            if not isinstance(server, Server) \
+                    or not isinstance(node, model.Node):
                 raise TypeError('First element must be of "Server" type, '
                                 'second must be of "Node" type.')
             server.add_node(node)
@@ -297,13 +310,15 @@ class StoryBoard:
 
     def start(self):
         self.start_time = time.time()
-        script = r'nohup "$@" > /tmp/$1 & echo "$!"'
+        script = r'logname="$1"; shift; nohup "${@}" ' \
+                 r'> /tmp/${logname}.rumba.log 2>&1  & echo "$!"'
         logger.debug("Writing utility startup script on client nodes.")
         for server in self.servers:
             for client in server.clients:
                 for node in client.nodes:
                     node.execute_command(
-                        "echo '%s' > startup.sh && chmod a+x startup.sh" % (script,)
+                        "echo '%s' > startup.sh && chmod a+x startup.sh"
+                        % (script,)
                     )
         try:
             for server in self.servers:
@@ -325,3 +340,25 @@ class StoryBoard:
                 client.stop()
             for server in self.servers:
                 server.stop()
+
+    def fetch_logs(self, local_dir='.'):
+        if not os.path.isdir(local_dir):
+            raise Exception('"%s" is not a directory. Cannot fetch logs.'
+                            % local_dir)
+        server_nodes = set()
+        client_nodes = set()
+        for server in self.servers:
+            for node in server.nodes:
+                server_nodes.add(node)
+            for client in server.clients:
+                for node in client.nodes:
+                    client_nodes.add(node)
+        for node in server_nodes:
+            logs_list = node.execute_command('ls *_server.log')
+            logger.info('Log list is:\n%s', logs_list)
+            node.fetch_files(logs_list.split('\n'), local_dir)
+        for node in client_nodes:
+            logs_list = node.execute_command('ls /tmp/*.rumba.log '
+                                             '|| echo ""')
+            logger.info('Log list is:\n%s', logs_list)
+            node.fetch_files(logs_list.split('\n'), local_dir)