Changeset 42cd8a7 for fedd/federation

Timestamp:

May 30, 2010 9:17:35 AM (14 years ago)

Author:

Ted Faber <faber@…>

Branches:

axis_example, compt_changes, info-ops, master, version-3.01, version-3.02

Children:

37ed9a5

Parents:

703859f

Message:

Remove the proxy split from protogeni. It gets in the way of code inheritance
and generally makes a mess.

Location:

fedd/federation

Files:

• protogeni_access.py (modified) (11 diffs)
• proxy_protogeni_segment.py (deleted)

fedd/federation/protogeni_access.py

@@ -10 +10 @@
 import logging
 import subprocess
+import random
 import traceback
 
-from threading import *
+from threading import Thread, Timer, Lock
 from M2Crypto.SSL import SSLError
 
@@ -27 +28 @@
 
 from access import access_base
+from proxy_segment import proxy_segment
 
 import topdl
 import list_log
-import proxy_protogeni_segment
 
 
@@ -39 +40 @@
 fl = logging.getLogger("fedd.access")
 fl.addHandler(nullHandler())
+
+class protogeni_proxy(proxy_segment):
+    class ProtoGENIError(Exception): 
+        def __init__(self, op, code, output):
+            Exception.__init__(self, output)
+            self.op = op
+            self.code = code
+            self.output = output
+
+    def __init__(self, log=None, keyfile=None, debug=False, 
+            ch_url=None, sa_url=None, cm_url=None):
+        proxy_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+
+        self.ProtoGENIError = protogeni_proxy.ProtoGENIError
+        self.ch_url = ch_url
+        self.sa_url = sa_url
+        self.cm_url = cm_url
+
+        self.call_SetValue = service_caller('SetValue')
+
+        self.debug_fail = ['Resolve']
+        self.debug_response = {
+                'RedeemTicket': ("XML blob1", "XML blob2"),
+                'SliceStatus': { 'status': 'ready' },
+            }
+
+
+    def pg_call(self, url, method, params, context):
+        max_retries = 5
+        retries = 0
+
+        s = service_caller(method, request_body_name="", strict=False)
+        self.log.debug("Calling %s %s" % (url, method))
+        if not self.debug:
+            while retries < max_retries:
+                r = s.call_xmlrpc_service(url, params, context=context)
+                code = r.get('code', -1)
+                if code == 0:
+                    # Success leaves the loop here
+                    return r.get('value', None)
+                elif code == 14 and retries +1 < max_retries:
+                    # Busy resource
+                    retries+= 1
+                    self.log.info("Resource busy, retrying in 30 secs")
+                    time.sleep(30)
+                else:
+                    # NB: out of retries falls through to here
+                    raise self.ProtoGENIError(op=method, 
+                            code=r.get('code', 'unknown'), 
+                            output=r.get('output', 'No output'))
+        else:
+            if method in self.debug_fail:
+                raise self.ProtoGENIError(op=method, code='unknown',
+                        output='No output')
+            elif self.debug_response.has_key(method):
+                return self.debug_response[method]
+            else:
+                return "%s XML blob" % method
+
+
 
 class access(access_base):
@@ -123 +184 @@
             self.read_access(config.get("access", "accessdb"), 
                     access_obj=self.make_access_info)
-
-        self.start_segment = proxy_protogeni_segment.start_segment
-        self.stop_segment = proxy_protogeni_segment.stop_segment
-        self.renew_segment = proxy_protogeni_segment.renew_segment
 
         self.lookup_access = self.lookup_access_base
@@ -308 +365 @@
             self.state_lock.release()
             raise service_error(service_error.req, "No such allocation")
+
+    # Turn the manifest into a dict were each virtual nodename (i.e. the topdl
+    # name) has an entry with the allocated machine in hostname and the
+    # interfaces in 'interfaces'.  I love having XML parser code lying around.
+    def manifest_to_dict(self, manifest, ignore_debug=False):
+        if self.create_debug and not ignore_debug: 
+            self.log.debug("Returning null manifest dict")
+            return { }
+
+        # The class allows us to keep a little state - the dict under
+        # consteruction and the current entry in that dict for the interface
+        # element code.
+        class manifest_parser:
+            def __init__(self):
+                self.d = { }
+                self.current_key=None
+
+            # If the element is a node, create a dict entry for it.  If it's an
+            # interface inside a node, add an entry in the interfaces list with
+            # the virtual name and component id.
+            def start_element(self, name, attrs):
+                if name == 'node':
+                    self.current_key = attrs.get('virtual_id',"")
+                    if self.current_key:
+                        self.d[self.current_key] = {
+                                'hostname': attrs.get('hostname', None),
+                                'interfaces': { }
+                                }
+                elif name == 'interface' and self.current_key:
+                    self.d[self.current_key]['interfaces']\
+                            [attrs.get('virtual_id','')] = \
+                            attrs.get('component_id', None)
+            #  When a node is finished, clear current_key
+            def end_element(self, name):
+                if name == 'node': self.current_key = None
+
+        node = { }
+
+        mp = manifest_parser()
+        p = xml.parsers.expat.ParserCreate()
+        # These are bound to the class we just created
+        p.StartElementHandler = mp.start_element
+        p.EndElementHandler = mp.end_element
+
+        p.Parse(manifest)
+        # Make the node dict that the callers expect
+        for k in mp.d:
+            node[k] = mp.d.get('hostname', '')
+        return mp.d
+
+    def fake_manifest(self, topo):
+        node = { }
+        for i, e in enumerate([ e for e in topo.elements \
+                if isinstance(e, topdl.Computer)]):
+            node[e.name] = { 
+                    'hostname': "node%03d" % i,
+                    'interfaces': { }
+                    }
+            for j, inf in enumerate(e.interface):
+                node[e.name]['interfaces'][inf.name] = 'eth%d' % j
+
+        return node
+
+
+    def generate_portal_configs(self, topo, pubkey_base, 
+            secretkey_base, tmpdir, leid, connInfo, services, nodes):
+
+        def conninfo_to_dict(key, info):
+            """
+            Make a cpoy of the connection information about key, and flatten it
+            into a single dict by parsing out any feddAttrs.
+            """
+
+            rv = None
+            for i in info:
+                if key == i.get('portal', "") or \
+                        key in [e.get('element', "") \
+                        for e in i.get('member', [])]:
+                    rv = i.copy()
+                    break
+
+            else:
+                return rv
+
+            if 'fedAttr' in rv:
+                for a in rv['fedAttr']:
+                    attr = a.get('attribute', "")
+                    val = a.get('value', "")
+                    if attr and attr not in rv:
+                        rv[attr] = val
+                del rv['fedAttr']
+            return rv
+
+        # XXX: un hardcode this
+        def client_null(f, s):
+            print >>f, "Service: %s" % s['name']
+        
+        def client_seer_master(f, s):
+            print >>f, 'PortalAlias: seer-master'
+
+        def client_smb(f, s):
+            print >>f, "Service: %s" % s['name']
+            smbshare = None
+            smbuser = None
+            smbproj = None
+            for a in s.get('fedAttr', []):
+                if a.get('attribute', '') == 'SMBSHARE':
+                    smbshare = a.get('value', None)
+                elif a.get('attribute', '') == 'SMBUSER':
+                    smbuser = a.get('value', None)
+                elif a.get('attribute', '') == 'SMBPROJ':
+                    smbproj = a.get('value', None)
+
+            if all((smbshare, smbuser, smbproj)):
+                print >>f, "SMBshare: %s" % smbshare
+                print >>f, "ProjectUser: %s" % smbuser
+                print >>f, "ProjectName: %s" % smbproj
+
+        def client_hide_hosts(f, s):
+            for a in s.get('fedAttr', [ ]):
+                if a.get('attribute', "") == 'hosts':
+                    print >>f, 'Hide: %s' % a.get('value', "")
+
+        client_service_out = {
+                'SMB': client_smb,
+                'tmcd': client_null,
+                'seer': client_null,
+                'userconfig': client_null,
+                'project_export': client_null,
+                'seer_master': client_seer_master,
+                'hide_hosts': client_hide_hosts,
+            }
+
+        def client_seer_master_export(f, s):
+            print >>f, "AddedNode: seer-master"
+
+        def client_seer_local_export(f, s):
+            print >>f, "AddedNode: control"
+
+        client_export_service_out = {
+                'seer_master': client_seer_master_export,
+                'local_seer_control': client_seer_local_export,
+            }
+
+        def server_port(f, s):
+            p = urlparse(s.get('server', 'http://localhost'))
+            print >>f, 'port: remote:%s:%s:%s' % (p.port, p.hostname, p.port) 
+
+        def server_null(f,s): pass
+
+        def server_seer(f, s):
+            print >>f, 'seer: true'
+
+        server_service_out = {
+                'SMB': server_port,
+                'tmcd': server_port,
+                'userconfig': server_null,
+                'project_export': server_null,
+                'seer': server_seer,
+                'seer_master': server_port,
+                'hide_hosts': server_null,
+            }
+        # XXX: end un hardcode this
+
+
+        seer_out = False
+        client_out = False
+        for e in [ e for e in topo.elements \
+                if isinstance(e, topdl.Computer) and e.get_attribute('portal')]:
+            myname = e.name
+            type = e.get_attribute('portal_type')
+
+            info = conninfo_to_dict(myname, connInfo)
+
+            if not info:
+                raise service_error(service_error.req,
+                        "No connectivity info for %s" % myname)
+
+            # Translate to physical name (ProtoGENI doesn't have DNS)
+            physname = nodes.get(myname, { }).get('hostname', None)
+            peer = info.get('peer', "")
+            ldomain = self.domain
+            ssh_port = info.get('ssh_port', 22)
+
+            # Collect this for the client.conf file
+            if 'masterexperiment' in info:
+                mproj, meid = info['masterexperiment'].split("/", 1)
+
+            active = info.get('active', 'False')
+
+            if type in ('control', 'both'):
+                testbed = e.get_attribute('testbed')
+                control_gw = myname
+
+            cfn = "%s/%s.gw.conf" % (tmpdir, myname.lower())
+            tunnelconfig = self.tunnel_config
+            try:
+                f = open(cfn, "w")
+                if active == 'True':
+                    print >>f, "active: True"
+                    print >>f, "ssh_port: %s" % ssh_port
+                    if type in ('control', 'both'):
+                        for s in [s for s in services \
+                                if s.get('name', "") in self.imports]:
+                            server_service_out[s['name']](f, s)
+
+                if tunnelconfig:
+                    print >>f, "tunnelip: %s" % tunnelconfig
+                print >>f, "peer: %s" % peer.lower()
+                print >>f, "ssh_pubkey: /usr/local/federation/etc/%s" % \
+                        pubkey_base
+                print >>f, "ssh_privkey: /usr/local/federation/etc/%s" % \
+                        secretkey_base
+                f.close()
+            except EnvironmentError, e:
+                raise service_error(service_error.internal,
+                        "Can't write protal config %s: %s" % (cfn, e))
+
+        # Done with portals, write the client config file.
+        try:
+            f = open("%s/client.conf" % tmpdir, "w")
+            if control_gw:
+                print >>f, "ControlGateway: %s" % physname.lower()
+            for s in services:
+                if s.get('name',"") in self.imports and \
+                        s.get('visibility','') == 'import':
+                    client_service_out[s['name']](f, s)
+                if s.get('name', '') in self.exports and \
+                        s.get('visibility', '') == 'export' and \
+                        s['name'] in client_export_service_out:
+                    client_export_service_out[s['name']](f, s)
+            # Seer uses this.
+            if mproj and meid:
+                print >>f, "ExperimentID: %s/%s" % (mproj, meid)
+            f.close()
+        except EnvironmentError, e:
+            raise service_error(service_error.internal,
+                    "Cannot write client.conf: %s" %s)
+
+
+
+    def export_store_info(self, cf, nodes, ssh_port, connInfo):
+        """
+        For the export requests in the connection info, install the peer names
+        at the experiment controller via SetValue calls.
+        """
+
+        for c in connInfo:
+            for p in [ p for p in c.get('parameter', []) \
+                    if p.get('type', '') == 'output']:
+
+                if p.get('name', '') == 'peer':
+                    k = p.get('key', None)
+                    surl = p.get('store', None)
+                    if surl and k and k.index('/') != -1:
+                        if self.create_debug:
+                            req = { 'name': k, 'value': 'debug' }
+                            self.call_SetValue(surl, req, cf)
+                        else:
+                            n = nodes.get(k[k.index('/')+1:], { })
+                            value = n.get('hostname', None)
+                            if value:
+                                req = { 'name': k, 'value': value }
+                                self.call_SetValue(surl, req, cf)
+                            else:
+                                self.log.error("No hostname for %s" % \
+                                        k[k.index('/'):])
+                    else:
+                        self.log.error("Bad export request: %s" % p)
+                elif p.get('name', '') == 'ssh_port':
+                    k = p.get('key', None)
+                    surl = p.get('store', None)
+                    if surl and k:
+                        req = { 'name': k, 'value': ssh_port }
+                        self.call_SetValue(surl, req, cf)
+                    else:
+                        self.log.error("Bad export request: %s" % p)
+                else:
+
+                    self.log.error("Unknown export parameter: %s" % \
+                            p.get('name'))
+                    continue
+
+    def configure_nodes(self, segment_commands, topo, nodes, user, pubkey, secretkey, 
+            stagingdir, tmpdir):
+
+        # These little functions/functors just make things more readable
+        class stage_file_type:
+            def __init__(self, user, host, stagingdir):
+                self.user = user
+                self.host = host
+                self.stagingdir = stagingdir
+                self.scp = "/usr/bin/scp -i .ssh/id_rsa -o " + \
+                        "'ForwardX11 no' -o 'StrictHostKeyChecking no' "
+
+            def __call__(self, script, file, dest="."):
+                # If the file is a full pathname, do not use stagingdir
+                if file.find('/') == -1:
+                    file = "%s/%s" % (self.stagingdir, file)
+                print >>script, "%s %s@%s:%s %s" % \
+                        (self.scp, self.user, self.host, file, dest)
+
+        def install_tar(script, loc, base):
+            tar = "/bin/tar"
+            mkdir="/bin/mkdir"
+
+            print >>script, "%s -p %s" % (mkdir, loc)
+            print >>script, "%s -C %s -xzf %s" % (tar, loc, base)
+
+        def install_rpm(script, base):
+            rpm = "/bin/rpm"
+            print >>script, "%s --install %s" % (rpm, base)
+
+        fed_dir = "/usr/local/federation"
+        fed_etc_dir = "%s/etc" % fed_dir
+        fed_bin_dir = "%s/bin" % fed_dir
+        fed_lib_dir = "%s/lib" % fed_dir
+
+        ifconfig = "/sbin/ifconfig"
+
+        stage_file = stage_file_type(user, self.staging_host, stagingdir)
+
+        for e in [ e for e in topo.elements if isinstance(e, topdl.Computer)]:
+            vname = e.name
+            node = nodes.get(vname, { })
+            pname = node.get('hostname', None)
+            if pname:
+                script = open("%s/%s.startup" %(tmpdir, pname), "w")
+                # Reset the interfaces to the ones in the topo file
+                for i in [ i for i in e.interface \
+                        if not i.get_attribute('portal')]:
+                    pinf = node['interfaces'].get(i.name, None)
+                    addr = i.get_attribute('ip4_address') 
+                    netmask = i.get_attribute('ip4_netmask') or '255.255.255.0'
+                    if pinf and addr:
+                        print >>script, \
+                                "%s %s %s netmask %s"  % \
+                                (ifconfig, pinf, addr, netmask)
+                    else:
+                        self.log.error("Missing interface or address for %s" \
+                                % i.name)
+                    
+                for l, f in self.federation_software:
+                    base = os.path.basename(f)
+                    stage_file(script, base)
+                    if l: install_tar(script, l, base)
+                    else: install_rpm(script, base)
+
+                for s in e.software:
+                    s_base = s.location.rpartition('/')[2]
+                    stage_file(script, s_base)
+                    if s.install: install_tar(script, s.install, s_base)
+                    else: install_rpm(script, s_base)
+
+                for f in ('hosts', pubkey, secretkey, 'client.conf', 
+                        'userconf'):
+                    stage_file(script, f, fed_etc_dir)
+                if self.sshd:
+                    stage_file(script, self.sshd, fed_bin_dir)
+                if self.sshd_config:
+                    stage_file(script, self.sshd_config, fed_etc_dir)
+
+                # Look in tmpdir to get the names.  They've all been copied
+                # into the (remote) staging dir
+                if os.access("%s/%s.gw.conf" % (tmpdir, vname), os.R_OK):
+                    stage_file(script, "%s.gw.conf" % vname, fed_etc_dir)
+
+                # Hackery dackery dock: the ProtoGENI python is really ancient.
+                # A modern version (though packaged for Mandrake (remember
+                # Mandrake?  good times, good times)) should be in the
+                # federation_software list, but we need to move rename is for
+                # SEER.
+                print >>script, "rm /usr/bin/python"
+                print >>script, "ln /usr/bin/python2.4 /usr/bin/python"
+                # Back to less hacky stuff
+
+                # Start commands
+                if e.get_attribute('portal') and self.portal_startcommand:
+                    # Install portal software
+                    for l, f in self.portal_software:
+                        base = os.path.basename(f)
+                        stage_file(script, base)
+                        if l: install_tar(script, l, base)
+                        else: install_rpm(script, base)
+
+                    # Portals never have a user-specified start command
+                    print >>script, self.portal_startcommand
+                elif self.node_startcommand:
+                    # XXX: debug
+                    print >>script, "sudo perl -I%s %simport_key.pl /users/%s/.ssh/authorized_keys /root/.ssh/authorized_keys" % (fed_lib_dir, fed_bin_dir, user)
+                    # XXX: debug
+                    if e.get_attribute('startup'):
+                        print >>script, "%s \\$USER '%s'" % \
+                                (self.node_startcommand, e.get_attribute('startup'))
+                    else:
+                        print >>script, self.node_startcommand
+                script.close()
+                if not segment_commands.scp_file("%s/%s.startup" % (tmpdir, pname), 
+                        user, pname):
+                    self.log.error("Could not copy script to %s" % pname)
+            else:
+                self.log.error("Unmapped node: %s" % vname)
+
+    def start_node(self, user, host, node, segment_commands):
+        # Place an identity on the node so that the copying can succeed
+        segment_commands.ssh_cmd(user, host, "scp .ssh/id_rsa %s:.ssh" % node)
+        segment_commands.ssh_cmd(user, node, 
+                "sudo /bin/sh ./%s.startup &" % node)
+
+    def start_nodes(self, user, host, nodes, segment_commands):
+        threads = [ ]
+        for n in nodes:
+            t = Thread(target=self.start_node, args=(user, host, n, 
+                segment_commands))
+            t.start()
+            threads.append(t)
+
+        done = [not t.isAlive() for t in threads]
+        while not all(done):
+            self.log.info("Waiting for threads %s" % done)
+            time.sleep(10)
+            done = [not t.isAlive() for t in threads]
+
+
+
+
+    def start_segment(self, segment_commands, aid, user, rspec, pubkey,
+            secretkey, ename, stagingdir, tmpdir, certfile, certpw,
+            export_certfile, topo, connInfo, services, timeout=0):
+        """
+        Start a sub-experiment on a federant.
+
+        Get the current state, modify or create as appropriate, ship data
+        and configs and start the experiment.  There are small ordering
+        differences based on the initial state of the sub-experiment.
+        """
+
+        def random_slicename(user):
+            slicename = user
+            for i in range(0,5):
+                slicename += random.choice(string.ascii_letters)
+            return slicename
+
+        host = self.staging_host
+        if not os.access(certfile, os.R_OK):
+            self.log.error("[start_segment]: Cannot read certfile: %s" % \
+                    certfile)
+            return False
+        ctxt = fedd_ssl_context(my_cert=certfile, password=certpw)
+        # Local software dir
+        lsoftdir = "%s/software" % tmpdir
+
+        # Open up a temporary file to contain a script for setting up the
+        # filespace for the new experiment.
+        self.log.info("[start_segment]: creating script file")
+        try:
+            sf, scriptname = tempfile.mkstemp()
+            scriptfile = os.fdopen(sf, 'w')
+        except EnvironmentError:
+            return False
+
+        scriptbase = os.path.basename(scriptname)
+
+        # Script the filesystem changes
+        print >>scriptfile, "/bin/rm -rf %s" % stagingdir
+        print >>scriptfile, 'mkdir -p %s' % stagingdir
+        print >>scriptfile, "rm -f %s" % scriptbase
+        scriptfile.close()
+
+        # Move the script to the remote machine
+        # XXX: could collide tempfile names on the remote host
+        if segment_commands.scp_file(scriptname, user, host, scriptbase):
+            os.remove(scriptname)
+        else:
+            return False
+
+        # Execute the script (and the script's last line deletes it)
+        if not segment_commands.ssh_cmd(user, host, "sh -x %s" % scriptbase):
+            return False
+
+        try:
+            gcred = segment_commands.pg_call(self.sa_url, 
+                    'GetCredential', {}, ctxt)
+        except self.ProtoGENIError, e:
+            raise service_error(service_error.federant,
+                    "ProtoGENI: %s" % e)
+        # Find a slicename not in use
+        slicename = "fabereGpgL"
+        while True:
+            slicename = random_slicename(user)
+            try:
+                param = {
+                        'credential': gcred, 
+                        'hrn': slicename,
+                        'type': 'Slice'
+                        }
+                segment_commands.pg_call(self.sa_url, 'Resolve', param, ctxt)
+            except segment_commands.ProtoGENIError, e:
+                print e
+                break
+
+        self.log.info("Creating %s" % slicename)
+        f = open("./rspec", "w")
+        print >>f, "%s" % rspec
+        f.close()
+        # Create the slice and allocate resources.  If any of this stuff fails,
+        # the allocations will time out on PG in short order, so we just raise
+        # the service_error.
+        try:
+            param = {
+                    'credential': gcred, 
+                    'hrn': slicename,
+                    'type': 'Slice'
+                    }
+            slice_cred = segment_commands.pg_call(self.sa_url, 'Register', param, ctxt)
+            f = open("./slice_cred", "w")
+            print >>f, slice_cred
+            f.close()
+            # Populate the ssh keys (let PG format them)
+            param = {
+                    'credential': gcred, 
+                    }
+            keys =  segment_commands.pg_call(self.sa_url, 'GetKeys', param, ctxt)
+            # Grab and redeem a ticket
+            param = {
+                    'credential': slice_cred, 
+                    'rspec': rspec,
+                    }
+            ticket = segment_commands.pg_call(self.cm_url, 'GetTicket', param, ctxt)
+            f = open("./ticket", "w")
+            print >>f, ticket
+            f.close()
+            param = { 
+                    'credential': slice_cred, 
+                    'keys': keys,
+                    'ticket': ticket,
+                    }
+            sliver_cred, manifest = segment_commands.pg_call(self.cm_url, 
+                    'RedeemTicket', param, ctxt)
+            f = open("./sliver_cred", "w")
+            print >>f, sliver_cred
+            f.close()
+            f = open("./manifest", "w")
+            print >>f, manifest
+            f.close()
+            # start 'em up
+            param = { 
+                    'credential': sliver_cred,
+                    }
+            segment_commands.pg_call(self.cm_url, 'StartSliver', param, ctxt)
+        except segment_commands.ProtoGENIError, e:
+            raise service_error(service_error.federant,
+                    "ProtoGENI: %s %s" % (e.code, e))
+
+        # With manifest in hand, we can export the portal node names.
+        if self.create_debug: nodes = self.fake_manifest(topo)
+        else: nodes = self.manifest_to_dict(manifest)
+
+        self.export_store_info(export_certfile, nodes, self.ssh_port,
+                connInfo)
+        self.generate_portal_configs(topo, pubkey, secretkey, tmpdir, 
+                ename, connInfo, services, nodes)
+
+        # Copy software to the staging machine (done after generation to copy
+        # those, too)
+        for d in (tmpdir, lsoftdir):
+            if os.path.isdir(d):
+                for f in os.listdir(d):
+                    if not os.path.isdir("%s/%s" % (d, f)):
+                        if not segment_commands.scp_file("%s/%s" % (d, f), 
+                                user, host, "%s/%s" % (stagingdir, f)):
+                            self.log.error("Scp failed")
+                            return False
+
+
+        # Now we wait for the nodes to start on PG
+        status = 'notready'
+        try:
+            while status == 'notready':
+                param = { 
+                        'credential': slice_cred
+                        }
+                r = segment_commands.pg_call(self.cm_url, 'SliceStatus', param, ctxt)
+                print r
+                status = r.get('status', 'notready')
+                if status == 'notready':
+                    time.sleep(30)
+        except segment_commands.ProtoGENIError, e:
+            raise service_error(service_error.federant,
+                    "ProtoGENI: %s %s" % (e.code, e))
+
+        if status == 'failed':
+            self.log.error('Sliver failed to start on ProtoGENI')
+            try:
+                param = { 
+                        'credential': slice_cred
+                        }
+                segment_commands.pg_call(self.cm_url, 'DeleteSliver', param, ctxt)
+            except segment_commands.ProtoGENIError, e:
+                raise service_error(service_error.federant,
+                    "ProtoGENI: %s" % e)
+            return False
+        else:
+            self.state_lock.acquire()
+            self.allocation[aid]['slice_name'] = slicename
+            self.allocation[aid]['slice_credential'] = slice_cred
+            self.allocation[aid]['sliver_credential'] = sliver_cred
+            self.allocation[aid]['manifest'] = manifest
+            self.allocation[aid]['certfile'] = certfile
+            self.allocation[aid]['certpw'] = certpw
+            self.write_state()
+            self.state_lock.release()
+
+        # Now we have configuration to do for ProtoGENI
+        self.configure_nodes(segment_commands, topo, nodes, user, pubkey, secretkey, 
+                stagingdir, tmpdir)
+
+        self.start_nodes(user, self.staging_host, 
+                [ n.get('hostname', None) for n in nodes.values()],
+                segment_commands)
+
+        # Everything has gone OK.
+        return True, dict([(k, n.get('hostname', None)) \
+                for k, n in nodes.items()])
 
     def generate_rspec(self, topo, softdir, connInfo):
@@ -457 +1138 @@
                 cpw, alloc_log)
 
-    def finalize_experiment(self, topo, starter, aid, alloc_id):
+    def finalize_experiment(self, topo, nodes, aid, alloc_id):
         # Copy the assigned names into the return topology
         rvtopo = topo.clone()
         embedding = [ ]
-        for n in starter.node:
+        for k, n in nodes.items():
             embedding.append({ 
-                'toponame': n,
-                'physname': ["%s%s" %  (starter.node[n], self.domain)],
+                'toponame': k,
+                'physname': ["%s%s" %  (n, self.domain)],
                 })
         # Grab the log (this is some anal locking, but better safe than
@@ -535 +1216 @@
                 cpw, alloc_log = self.initialize_experiment_info(attrs,
                         aid, certfile, tmpdir)
-            # XXX: we really need to put the import and connection info
-            # generation off longer.
             self.import_store_info(certfile, connInfo)
             rspec = self.generate_rspec(topo, "%s/%s/" \
                     % (self.staging_dir, ename), connInfo)
 
-            starter = self.start_segment(keyfile=ssh_key,
+            segment_commands = protogeni_proxy(keyfile=ssh_key,
                     debug=self.create_debug, log=alloc_log,
                     ch_url = self.ch_url, sa_url=self.sa_url,
                     cm_url=self.cm_url)
-            rv = starter(self, aid, user, rspec, pubkey_base, secretkey_base,
-                    ename,
+            rv, nodes = self.start_segment(segment_commands, aid, user, rspec,
+                    pubkey_base, 
+                    secretkey_base, ename,
                     "%s/%s" % (self.staging_dir, ename), tmpdir, cf, cpw,
                     certfile, topo, connInfo, services)
@@ -563 +1243 @@
 
         if rv:
-            return self.finalize_experiment(topo, starter, aid, req['allocID'])
+            return self.finalize_experiment(topo, nodes, aid, req['allocID'])
         elif err:
             raise service_error(service_error.federant,
@@ -569 +1249 @@
         else:
             raise service_error(service_error.federant, "Swapin failed")
+
+    def stop_segment(self, segment_commands, user, stagingdir, slice_cred,
+            certfile, certpw):
+        """
+        Stop a sub experiment by calling swapexp on the federant
+        """
+        host = self.staging_host
+        rv = False
+        try:
+            # Clean out tar files: we've gone over quota in the past
+            if stagingdir:
+                segment_commands.ssh_cmd(user, host, "rm -rf %s" % stagingdir)
+            if slice_cred:
+                self.log.error('Removing Sliver on ProtoGENI')
+                ctxt = fedd_ssl_context(my_cert=certfile, password=certpw)
+                try:
+                    param = { 
+                            'credential': slice_cred
+                            }
+                    segment_commands.pg_call(self.cm_url, 'DeleteSlice',
+                            param, ctxt)
+                except segment_commands.ProtoGENIError, e:
+                    raise service_error(service_error.federant,
+                        "ProtoGENI: %s" % e)
+            return True
+        except self.ssh_cmd_timeout:
+            rv = False
+        return rv
 
     def TerminateSegment(self, req, fid):
@@ -599 +1307 @@
             staging = None
 
-        stopper = self.stop_segment(keyfile=ssh_key, debug=self.create_debug,
-                ch_url = self.ch_url, sa_url=self.sa_url, cm_url=self.cm_url)
-        stopper(self, user, staging, slice_cred, cf, cpw)
+        segment_commands = protogeni_proxy(keyfile=ssh_key,
+                debug=self.create_debug, ch_url = self.ch_url,
+                sa_url=self.sa_url, cm_url=self.cm_url)
+        self.stop_segment(segment_commands, user, staging, slice_cred, cf, cpw)
         return { 'allocID': req['allocID'] }
+
+    def renew_segment(self, segment_commands, name, scred, interval, 
+            certfile, certpw):
+        ctxt = fedd_ssl_context(my_cert=certfile, password=certpw)
+        try:
+            expiration = time.strftime("%Y%m%dT%H:%M:%S",
+                    time.gmtime(time.time() + interval))
+            cred = segment_commands.pg_call(self.sa_url, 'GetCredential', {}, ctxt)
+
+            param = {
+                    'credential': scred,
+                    'expiration': expiration
+                    }
+            r = segment_commands.pg_call(self.sa_url, 'RenewSlice', param, ctxt)
+            param = {
+                    'credential': cred,
+                    'hrn': name,
+                    'type': 'Slice',
+                    }
+            slice = segment_commands.pg_call(self.sa_url, 'Resolve', param, ctxt)
+            uuid = slice.get('uuid', None)
+            if uuid == None:
+                sys.exit('No uuid for %s' % slicename)
+
+            print 'Calling GetCredential (uuid)'
+            param = {
+                    'credential': cred,
+                    'uuid': uuid,
+                    'type': 'Slice',
+                    }
+            new_scred = segment_commands.pg_call(self.sa_url, 'GetCredential', param, ctxt)
+            f = open('./new_slice_cred', 'w')
+            print >>f, new_scred
+            f.close()
+
+        except segment_commands.ProtoGENIError, e:
+            self.log.error("Failed to extend slice %s: %s" % (name, e))
+            return None
+        try:
+            print 'Calling RenewSlice (CM)'
+            param = {
+                    'credential': new_scred,
+                    }
+            r = segment_commands.pg_call(self.cm_url, 'RenewSlice', param, ctxt)
+        except segment_commands.ProtoGENIError, e:
+            self.log.warn("Failed to renew sliver for %s: %s" % (name, e))
+
+        return new_scred
+    
 
     def RenewSlices(self):
@@ -628 +1386 @@
             # There's a ProtoGENI slice associated with the segment; renew it.
             if name and scred:
-                renewer = self.renew_segment(log=self.log, 
+                segment_commands = protogeni_proxy(log=self.log, 
                         debug=self.create_debug, keyfile=ssh_key,
                         cm_url = self.cm_url, sa_url = self.sa_url,
                         ch_url = self.ch_url)
-                new_scred = renewer(name, scred, self.renewal_interval, cf, cpw)
+                new_scred = self.renew_segment(segment_commands, name, scred, 
+                        self.renewal_interval, cf, cpw)
                 if new_scred:
                     self.log.info("Slice %s renewed until %s GMT" % \