Changeset 06c1dba88c7544851f08b5817f4ea0064ed4fefc

Timestamp:

01/09/12 16:05:27 (4 months ago)

Author:

Ted Faber <faber@…>

Children:

c7141dc6739a15071ba1ae8386e6457078c1cc41

Parents:

7f57435a6dc5f5eb9336148c09eb218296e23659

git-committer:

Ted Faber <faber@isi.edu> / 2012-01-09T16:05:27Z-0800

Message:

Fisrt step to multi-user - checkpoint

Location:

fedd/federation

Files:

• emulab_access.py (modified) (2 diffs)
• emulab_segment.py (modified) (1 diff)
• local_emulab_segment.py (deleted)
• ssh_emulab_segment.py (moved) (moved from fedd/federation/proxy_segment.py) (1 diff)
• xmlrpc_emulab_segment.py (added)

fedd/federation/emulab_access.py

@@ -32 +32 @@
 import topdl
 import list_log
-import proxy_emulab_segment
-import local_emulab_segment
+import emulab_segment
 
 
@@ -98 +97 @@
 
         self.access_type = self.access_type.lower()
-        if self.access_type == 'remote_emulab':
-            self.start_segment = proxy_emulab_segment.start_segment
-            self.stop_segment = proxy_emulab_segment.stop_segment
-            self.info_segment = proxy_emulab_segment.info_segment
-            self.operation_segment = proxy_emulab_segment.operation_segment
-        elif self.access_type == 'local_emulab':
-            self.start_segment = local_emulab_segment.start_segment
-            self.stop_segment = local_emulab_segment.stop_segment
-            self.info_segment = local_emulab_segment.info_segment
-            self.operation_segment = local_emulab_segment.operation_segment
-        else:
-            self.start_segment = None
-            self.stop_segment = None
-            self.info_segment = None
-            self.operation_segment = None
+        self.start_segment = emulab_segment.start_segment
+        self.stop_segment = emulab_segment.stop_segment
+        self.info_segment = emulab_segment.info_segment
+        self.operation_segment = emulab_segment.operation_segment
 
         self.restricted = [ ]

fedd/federation/emulab_segment.py

@@ -1 +1 @@
 #!/usr/local/bin/python
 
+import sys, os
+import re
+
+import tempfile
+import subprocess
 import logging 
+import time
+import signal
+
 import util
-import topdl
 
-from M2Crypto import SSL
-from M2Crypto.m2xmlrpclib import SSL_Transport
-from xmlrpclib import ServerProxy, dumps, loads, Fault, Error, Binary
-from M2Crypto.SSL import SSLError
-from M2Crypto.BIO import BIOError
-from socket import error as socket_error
-from socket import sslerror
-import httplib
-from federation.util import fedd_ssl_context
-from federation import service_error
-from federation.operation_status import operation_status
+from ssh_emulab_segment import ssh_emulab_segment
+from xmlrpc_emulab_segment import xmlrpc_emulab_segment
 
-class emulab_segment:
-    class node_info:
-        def __init__(self, name, pname, status=None, osname=None, 
-                osversion=None, image=None):
-            self.name = name
-            self.pname = pname
-            self.status = status
-            self.osname = osname
-            self.osversion = osversion
-            self.image = image
-        def getOS(self):
-            rv = None
-            if self.osname or self.osversion:
-                rv = topdl.OperatingSystem(name=self.osname, 
-                        version=self.osversion)
-            if self.image and rv:
-                rv.set_attribute('emulab_access:image', self.image)
-            return rv
 
-    def __init__(self, boss, cert):
-        self.ctxt = fedd_ssl_context(my_cert=cert)
-        self.ctxt.set_verify(SSL.verify_none, 10)
-        self.boss = boss
-        self.null = """
-set ns [new Simulator]
-source tb_compat.tcl
+class start_segment(ssh_emulab_segment, xmlrpc_emulab_segment):
+    def __init__(self, log=None, keyfile=None, debug=False, boss=None,
+            cert=None):
+        ssh_emulab_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+        xmlrpc_emulab_segment.__init__(self, boss=boss, cert=cert)
 
-set a [$ns node]
+    def set_up_experiment_filespace(self, user, host, pid, eid, tmpdir):
+        """
+        Send all the software and configuration files into the experiment's
+        file space.  To reduce the number of ssh connections, we script many
+        changes and execute the script.
+        """
+        # Configuration directories on the remote machine
+        proj_dir = "/proj/%s/exp/%s/tmp" % (pid, eid)
+        softdir = "/proj/%s/software/%s" % (pid, eid)
+        # Local software dir
+        lsoftdir = "%s/software" % tmpdir
 
-$ns rtproto Session
-$ns run 
-"""
-        self.log =  getattr(self, 'log', None) 
-        self.debug = getattr(self, 'debug', False)
-        self.node = { }
-        self.status = [ ]
-        self.node_info = emulab_segment.node_info
+        # Open up a temporary file to contain a script for setting up the
+        # filespace for the new experiment.
+        self.log.info("[start_segment]: creating script file")
+        try:
+            sf, scriptname = tempfile.mkstemp()
+            scriptfile = os.fdopen(sf, 'w')
+        except EnvironmentError:
+            return False
 
-    def emulab_call(self, method, params):
-        VERSION = 0.1
-        try:
-            transport = SSL_Transport(self.ctxt)
-            port = ServerProxy(self.boss, transport=transport)
-            remote_method = getattr(port, method, None)
-            if remote_method is not None:
-                resp = remote_method(VERSION, params)
-            else:
-                raise service_error(service_error.internal, 
-                        "Bad method: %s" % method)
-        except socket_error, e:
-            raise service_error(service_error.connect, 
-                    "Cannot connect" % e)
-            raise e
-        except BIOError, e:
-            if self.log:
-                self.log.warn("BIO error: %s" % e)
-            raise e
-        except sslerror, e:
-            if self.log:
-                self.log.warn("SSL (socket) error: %s" %  e)
-            raise e
-        except SSLError, e:
-            if self.log:
-                self.log.warn("SSL error: %s" % e)
-            raise e
-        except httplib.HTTPException, e:
-            if self.log:
-                self.log.warn("HTTP error: %s" % e)
-            raise e
-        except Fault, f:
-            raise service_error(service_error.protocol, 
-                    "Remote XMLRPC Fault: %s" % f)
-        except Error, e:
-            raise service_error(service_error.protocol, 
-                    "Remote XMLRPC Fault: %s" % e)
+        scriptbase = os.path.basename(scriptname)
 
-        code = resp.get('code', -1)
-        if code ==0:
-            return (code, resp.get('value', None))
+        # Script the filesystem changes
+        print >>scriptfile, "/bin/rm -rf %s" % proj_dir
+        # Clear and create the software directory
+        print >>scriptfile, "/bin/rm -rf %s/*" % softdir
+        print >>scriptfile, 'mkdir -p %s' % proj_dir
+        if os.path.isdir(lsoftdir):
+            print >>scriptfile, 'mkdir -p %s' % softdir
+        print >>scriptfile, "rm -f %s" % scriptbase
+        scriptfile.close()
+
+        # Move the script to the remote machine
+        # XXX: could collide tempfile names on the remote host
+        if self.scp_file(scriptname, user, host, scriptbase):
+            os.remove(scriptname)
         else:
-            return (code, resp.get('output', None))
+            return False
 
-    def get_state(self, pid, eid):
+        # Execute the script (and the script's last line deletes it)
+        if not self.ssh_cmd(user, host, "sh -x %s" % scriptbase):
+            return False
+
+        for f in os.listdir(tmpdir):
+            if not os.path.isdir("%s/%s" % (tmpdir, f)):
+                if not self.scp_file("%s/%s" % (tmpdir, f), user, host,
+                        "%s/%s" % (proj_dir, f)):
+                    return False
+        if os.path.isdir(lsoftdir):
+            for f in os.listdir(lsoftdir):
+                if not os.path.isdir("%s/%s" % (lsoftdir, f)):
+                    if not self.scp_file("%s/%s" % (lsoftdir, f), 
+                            user, host, "%s/%s" % (softdir, f)):
+                        return False
+        return True
+
+
+    def __call__(self, parent, eid, pid, user, tclfile, tmpdir, timeout=0, 
+            gid=None):
         """
-        Return the state of the experiment as reported by emulab
+        Start a sub-experiment on a federant.
+
+        Get the current state, and terminate the experiment if it exists. The
+        group membership of the experiment is difficult to determine or change,
+        so start with a clean slate.  Create a new one and ship data
+        and configs and start the experiment.  There are small ordering
+        differences based on the initial state of the sub-experiment.
         """
 
-        if self.debug:
-            state = 'swapped'
-        else:
-            params =  { 'proj': pid, 'exp': eid }
-            code, state = self.emulab_call('experiment.state', params)
-            if code != 0:
-                state = 'none'
+        state = self.get_state(pid, eid)
 
-        if self.log:
-            self.log.debug("State is %s" % state)
-        return state
+        if state != 'none':
+            self.terminate_exp(pid, eid)
 
-    def make_null_experiment(self, pid, eid, tmpdir, gid=None):
+        if not self.make_null_experiment(pid, eid, tmpdir, gid):
+            return False
+
+        if not self.set_up_experiment_filespace(pid, eid, tmpdir):
+            return False
+
+        # Put the file into a string to pass to emulab.
+        try:
+            tcl = "".join([ l for l in open(tclfile,"r")])
+        except EnvironmentError, e:
+            self.log.error("Can't read %s: %s" % (tclfile, e))
+            return False
+        
+        # Stage the new configuration
+        if not self.modify_exp(pid, eid, tcl):
+            self.log.error("modify failed")
+            return False
+
+        if not self.swap_exp(pid, eid, 'in'):
+            self.log.error("swap in failed")
+            return False
+        # Everything has gone OK.
+        self.get_mapping(pid,eid)
+        return True
+
+class stop_segment(local_segment, emulab_segment):
+    def __init__(self, log=None, keyfile=None, debug=False, boss=None, 
+            cert=None):
+        local_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+        emulab_segment.__init__(self, boss=boss, cert=cert)
+
+    def __call__(self, parent, user, pid, eid, gid=None, terminate=False):
         """
-        Create a null copy of the experiment so that we capture any logs there
-        if the modify fails.  Emulab software discards the logs from a failed
-        startexp.
+        Stop a sub experiment by calling swapexp on the federant
         """
 
-        if self.debug:
-            if self.log:
-                self.log.debug("[make_null_experiment]: " + \
-                        "(debug) Creating experiment")
-            return True
-        else:
-            params = {
-                    'proj': pid,
-                    'exp': eid, 
-                    'nsfilestr': self.null,
-                    'batch': False,
-                    'idleswap': 0,
-                    'noidleswap_reason': 'Federated experiment',
-                    'noswapin': True,
-                    'wait': True
-                    }
-            if gid is not None:
-                params['group'] = gid
-            if self.log:
-                self.log.info("[make_null_experiment]: Creating experiment")
-            code, value = self.emulab_call('experiment.startexp', params)
+        self.log.info("[stop_segment]: Stopping %s" % eid)
+        rv = False
+        try:
+            # Clean out tar files: we've gone over quota in the past
+            self.ssh_cmd("rm -rf /proj/%s/software/%s" % (pid, eid))
+            rv = self.swap_exp(pid, eid, 'out')
+            if terminate:
+                rv = self.terminate_exp(pid, eid)
+        except self.cmd_timeout:
+            rv = False
+        return rv
 
-            if self.log:
-                if code == 0: 
-                    self.log.info('[make_null_experiment]: Create succeeded')
-                else: 
-                    self.log.error('[make_null_experiment]: Create failed: %s' \
-                            % value)
+class info_segment(local_segment, emulab_segment):
+    def __init__(self, log=None, keyfile=None, debug=False, boss=None, 
+            cert=None):
+        local_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+        emulab_segment.__init__(self, boss=boss, cert=cert)
 
-            return code == 0
+    def __call__(self, parent, user, pid, eid):
+        self.log.info("[info_segment]: Getting info from %s" % eid)
+        self.get_mapping(pid,eid)
+        return True
 
-    def swap_exp(self, pid, eid, direction='out', wait=True):
-        """
-        Swap experiment in.
-        """
-        if self.debug:
-            if self.log:
-                self.log.info("[swap_exp]: (debug) Swapping %s %s" % \
-                        (eid, direction))
-            return True
-        else:
-            if self.log:
-                self.log.info("[swap_exp]: Swapping %s %s" % (eid, direction))
-            params = {
-                    'proj': pid,
-                    'exp': eid,
-                    'direction': direction,
-                    'wait': wait,
-                    }
-            code, value = self.emulab_call('experiment.swapexp', params)
+class operation_segment(local_segment, emulab_segment):
+    def __init__(self, log=None, keyfile=None, debug=False, boss=None, 
+            cert=None):
+        local_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+        emulab_segment.__init__(self, boss=boss, cert=cert)
 
-            if self.log:
-                if code == 0: self.log.info('[swap_exp]: Swap succeeded')
-                else: self.log.error('[swap_exp]: Swap failed: %s' % value)
-
-        return code == 0
-
-    def terminate_exp(self, pid, eid, wait=True):
-        """
-        Completely terminate experiment
-        """
-        if self.debug:
-            if self.log:
-                self.log.info("[swap_exp]: (debug) terminate %s" %  eid)
-            return True
-        else:
-            if self.log:
-                self.log.info("[swap_exp]: Terminating %s" % eid)
-            params = {
-                    'proj': pid,
-                    'exp': eid,
-                    'wait': wait,
-                    }
-            code, value = self.emulab_call('experiment.endexp', params)
-
-            if self.log:
-                if code == 0: self.log.info('[swap_exp]: Terminate succeeded')
-                else: self.log.error('[swap_exp]: Terminate failed: %s' % value)
-
-        return code == 0
-
-    def modify_exp(self, pid, eid, tcl, wait=True):
-        if self.debug:
-            self.log.info("[modify_exp]: (debug) Modifying %s" % eid)
-            return True
-        else:
-            self.log.info("[modify_exp]: Modifying %s" % eid)
-            params = {
-                    'proj': pid,
-                    'exp': eid,
-                    'nsfilestr': tcl,
-                    'wait': wait,
-                    'reboot': True,
-                    'restart_eventsys': True,
-                    }
-            code, value = self.emulab_call('experiment.modify', params)
-            if self.log:
-                if code == 0: 
-                    self.log.info('[modify_exp]: Modify succeeded')
-                else: 
-                    self.log.error('[modify_exp]: Modify failed: %s' \
-                            % value)
-            return code == 0
-
-    def get_osid_map(self):
-        oses = { }
-        code, osids = self.emulab_call('osid.getlist', {})
-        for key, val in osids.items():
-            val['imageid'] = key
-            oses[val['osid']] = val
-        return oses
-
-    def get_mapping(self, pid, eid):
-        """
-        Get the physical to virtual mapping from the expinfo command and save
-        it in the self.map member.
-        """
-
-        ev_active = ('ISUP', 'ALWAYSUP' )
-        ev_starting = ('REBOOTED', 'REBOOTING','PXEBOOTING', 
-                'BOOTING', 'RELOADSETUP', 'RELOADING', 'RELOADDONE', 
-                'RELOADDONEV2', 'TBSETUP')
-        ev_terminating = ( 'SHUTDOWN' )
-
-
-        if self.debug:
-            if self.log:
-                self.log.info("[get_mapping] (debug) Generating mapping")
-                return True
-        else:
-            if self.log:
-                self.log.info("[get_mapping] Generating mapping")
-
-            osidmap = self.get_osid_map()
-
-            params = {
-                    'proj': pid,
-                    'exp': eid,
-                    'aspect': 'mapping'
-                    }
-            code, nodes = self.emulab_call('experiment.info', params)
-            if code ==0:
-                for k, v in nodes.items():
-                    if v.get('erole', False) and 'pnode' in v:
-                        st = v.get('status', 'up')
-                        ev = v.get('eventstatus', 'ISUP')
-                        os = v.get('osid', None)
-
-                        if st == 'up':
-                            if ev in ev_active: st = 'active'
-                            elif ev in ev_starting: st = 'starting'
-                            elif ev in ev_terminating: st = 'terminating'
-                            else: st = 'failed'
-                        else: st = 'failed'
-
-                        if os and os in osidmap:
-                           osname = osidmap[os].get('OS', None)
-                           osversion = osidmap[os].get('version', None)
-                           osimage = "%s/%s" % \
-                                   ( osidmap[os].get('pid', ''), 
-                                           osidmap[os].get('imageid', ''))
-                        else:
-                            osname = osversion = osimage = None
-
-                        self.node[k] = self.node_info(k, v['pnode'], st,
-                                osname, osversion, osimage)
-                if self.log:
-                    self.log.info("Mapping complete")
-                return True
-            else:
-                raise service_error(service_error.internal,
-                        "Cannot get node mapping of segment:%s/%s" % (pid, eid))
-    def get_initial_image(self, node, top):
-        for e in top.elements:
-            if isinstance(e, topdl.Computer):
-                if node == e.name:
-                    if e.os and len(e.os) == 1: 
-                        return e.os[0].get_attribute(
-                                'emulab_access:initial_image')
-        return None
-
-
-    def do_operation(self, op, lnode, pnode, params, top):
-        """
-        Carry out operation on node in the given experiment.
-        """
-        def get_param(params, name):
-            if params is None:
-                return None
-            for d in params:
-                if 'attribute' in d and d['attribute'] == name:
-                    return d.get('value', None)
-            else:
-                return None
-
-        op = op.lower()
-
-        if op == 'restore':
-            state = get_param(params, 'state')
-            if state is None:
-                self.status.append(operation_status(lnode, 
-                        operation_status.bad_param, 'No state to restore'))
-                return False
-            elif state == 'initial': 
-                image = self.get_initial_image(lnode, top)
-                if image:
-                    pid, iid = image.split('/')
-                    p = {'nodes': pnode, 'imagename': iid, 'imageproj': pid, 
-                            'wait': False}
-                    code, result = self.emulab_call('node.reload', p)
-                    if code == 0: 
-                        self.status.append(operation_status(lnode,
-                            operation_status.success, 'reloading'))
-                        return True
-                    else:
-                        self.status.append(operation_status(lnode,
-                                operation_status.federant, 
-                                'Error code: %d' % code))
-                        return False
-                else:
-                    self.status.append(operation_status(lnode,
-                            operation_status.federant, 
-                            'cannot find imageid??'))
-                    return False
-            elif state == 'boot': 
-                p = {'nodes': pnode, 'wait': False}
-                code, result = self.emulab_call('node.reboot', p)
-                if code == 0: 
-                    self.status.append(operation_status(lnode,
-                        operation_status.success, 'rebooting'))
-                    return True
-                else:
-                    self.status.append(operation_status(lnode,
-                            operation_status.federant, 'Error code: %d' % code))
-                    return False
-            else: 
-                if '/' in state:
-                    pid, iid = state.split('/')
-                else:
-                    pid = 'emulab-ops'
-                    iid = state
-
-                p = {'nodes': pnode, 'imagename': iid, 'imageproj': pid, 
-                        'wait': False}
-                code, result = self.emulab_call('node.reload', p)
-                if code == 0: 
-                    self.status.append(operation_status(lnode,
-                        operation_status.success, 'reloading'))
-                    return True
-                else:
-                    self.status.append(operation_status(lnode,
-                            operation_status.federant, 
-                            'Error code: %d' % code))
-                    return False
-        else: 
-            self.status.append(operation_status(lnode, operation_status.unsupp))
-            return False
-
+    def __call__(self, parent, op, targets, param, top):
+        for l, p in targets.items():
+            self.log.info("[operation_segment]: Calling op %s on %s(%s)" % \
+                    (op, l,p))
+            self.do_operation(op, l, p, param, top)
+        return True

fedd/federation/ssh_emulab_segment.py

@@ -12 +12 @@
 from service_error import service_error
 
-class proxy_segment:
+class ssh_emulab_segment:
     """
     Base class for segment starter classes that access their underlying testebd