Index: fedd/federation/protogeni_access.py
===================================================================
--- fedd/federation/protogeni_access.py	(revision 3551ae16efcbff61f2a479021f7b323bac7c2e11)
+++ fedd/federation/protogeni_access.py	(revision 42cd8a754077fa679c180043a1da9080d486cc29)
@@ -10,7 +10,8 @@
 import logging
 import subprocess
+import random
 import traceback
 
-from threading import *
+from threading import Thread, Timer, Lock
 from M2Crypto.SSL import SSLError
 
@@ -27,8 +28,8 @@
 
 from access import access_base
+from proxy_segment import proxy_segment
 
 import topdl
 import list_log
-import proxy_protogeni_segment
 
 
@@ -39,4 +40,64 @@
 fl = logging.getLogger("fedd.access")
 fl.addHandler(nullHandler())
+
+class protogeni_proxy(proxy_segment):
+    class ProtoGENIError(Exception): 
+	def __init__(self, op, code, output):
+	    Exception.__init__(self, output)
+	    self.op = op
+	    self.code = code
+	    self.output = output
+
+    def __init__(self, log=None, keyfile=None, debug=False, 
+	    ch_url=None, sa_url=None, cm_url=None):
+	proxy_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+
+	self.ProtoGENIError = protogeni_proxy.ProtoGENIError
+	self.ch_url = ch_url
+	self.sa_url = sa_url
+	self.cm_url = cm_url
+
+	self.call_SetValue = service_caller('SetValue')
+
+	self.debug_fail = ['Resolve']
+	self.debug_response = {
+		'RedeemTicket': ("XML blob1", "XML blob2"),
+		'SliceStatus': { 'status': 'ready' },
+	    }
+
+
+    def pg_call(self, url, method, params, context):
+	max_retries = 5
+	retries = 0
+
+	s = service_caller(method, request_body_name="", strict=False)
+	self.log.debug("Calling %s %s" % (url, method))
+	if not self.debug:
+	    while retries < max_retries:
+		r = s.call_xmlrpc_service(url, params, context=context)
+		code = r.get('code', -1)
+		if code == 0:
+		    # Success leaves the loop here
+		    return r.get('value', None)
+		elif code == 14 and retries +1 < max_retries:
+		    # Busy resource
+		    retries+= 1
+		    self.log.info("Resource busy, retrying in 30 secs")
+		    time.sleep(30)
+		else:
+		    # NB: out of retries falls through to here
+		    raise self.ProtoGENIError(op=method, 
+			    code=r.get('code', 'unknown'), 
+			    output=r.get('output', 'No output'))
+	else:
+	    if method in self.debug_fail:
+		raise self.ProtoGENIError(op=method, code='unknown',
+			output='No output')
+	    elif self.debug_response.has_key(method):
+		return self.debug_response[method]
+	    else:
+		return "%s XML blob" % method
+
+
 
 class access(access_base):
@@ -123,8 +184,4 @@
             self.read_access(config.get("access", "accessdb"), 
 		    access_obj=self.make_access_info)
-
-	self.start_segment = proxy_protogeni_segment.start_segment
-	self.stop_segment = proxy_protogeni_segment.stop_segment
-	self.renew_segment = proxy_protogeni_segment.renew_segment
 
 	self.lookup_access = self.lookup_access_base
@@ -308,4 +365,628 @@
 	    self.state_lock.release()
 	    raise service_error(service_error.req, "No such allocation")
+
+    # Turn the manifest into a dict were each virtual nodename (i.e. the topdl
+    # name) has an entry with the allocated machine in hostname and the
+    # interfaces in 'interfaces'.  I love having XML parser code lying around.
+    def manifest_to_dict(self, manifest, ignore_debug=False):
+	if self.create_debug and not ignore_debug: 
+	    self.log.debug("Returning null manifest dict")
+	    return { }
+
+	# The class allows us to keep a little state - the dict under
+	# consteruction and the current entry in that dict for the interface
+	# element code.
+	class manifest_parser:
+	    def __init__(self):
+		self.d = { }
+		self.current_key=None
+
+	    # If the element is a node, create a dict entry for it.  If it's an
+	    # interface inside a node, add an entry in the interfaces list with
+	    # the virtual name and component id.
+	    def start_element(self, name, attrs):
+		if name == 'node':
+		    self.current_key = attrs.get('virtual_id',"")
+		    if self.current_key:
+			self.d[self.current_key] = {
+				'hostname': attrs.get('hostname', None),
+				'interfaces': { }
+				}
+		elif name == 'interface' and self.current_key:
+		    self.d[self.current_key]['interfaces']\
+			    [attrs.get('virtual_id','')] = \
+			    attrs.get('component_id', None)
+	    #  When a node is finished, clear current_key
+	    def end_element(self, name):
+		if name == 'node': self.current_key = None
+
+	node = { }
+
+	mp = manifest_parser()
+	p = xml.parsers.expat.ParserCreate()
+	# These are bound to the class we just created
+	p.StartElementHandler = mp.start_element
+	p.EndElementHandler = mp.end_element
+
+	p.Parse(manifest)
+	# Make the node dict that the callers expect
+	for k in mp.d:
+	    node[k] = mp.d.get('hostname', '')
+	return mp.d
+
+    def fake_manifest(self, topo):
+	node = { }
+	for i, e in enumerate([ e for e in topo.elements \
+		if isinstance(e, topdl.Computer)]):
+	    node[e.name] = { 
+		    'hostname': "node%03d" % i,
+		    'interfaces': { }
+		    }
+	    for j, inf in enumerate(e.interface):
+		node[e.name]['interfaces'][inf.name] = 'eth%d' % j
+
+	return node
+
+
+    def generate_portal_configs(self, topo, pubkey_base, 
+	    secretkey_base, tmpdir, leid, connInfo, services, nodes):
+
+	def conninfo_to_dict(key, info):
+	    """
+	    Make a cpoy of the connection information about key, and flatten it
+	    into a single dict by parsing out any feddAttrs.
+	    """
+
+	    rv = None
+	    for i in info:
+		if key == i.get('portal', "") or \
+			key in [e.get('element', "") \
+			for e in i.get('member', [])]:
+		    rv = i.copy()
+		    break
+
+	    else:
+		return rv
+
+	    if 'fedAttr' in rv:
+		for a in rv['fedAttr']:
+		    attr = a.get('attribute', "")
+		    val = a.get('value', "")
+		    if attr and attr not in rv:
+			rv[attr] = val
+		del rv['fedAttr']
+	    return rv
+
+	# XXX: un hardcode this
+	def client_null(f, s):
+	    print >>f, "Service: %s" % s['name']
+	
+	def client_seer_master(f, s):
+	    print >>f, 'PortalAlias: seer-master'
+
+	def client_smb(f, s):
+	    print >>f, "Service: %s" % s['name']
+	    smbshare = None
+	    smbuser = None
+	    smbproj = None
+	    for a in s.get('fedAttr', []):
+		if a.get('attribute', '') == 'SMBSHARE':
+		    smbshare = a.get('value', None)
+		elif a.get('attribute', '') == 'SMBUSER':
+		    smbuser = a.get('value', None)
+		elif a.get('attribute', '') == 'SMBPROJ':
+		    smbproj = a.get('value', None)
+
+	    if all((smbshare, smbuser, smbproj)):
+		print >>f, "SMBshare: %s" % smbshare
+		print >>f, "ProjectUser: %s" % smbuser
+		print >>f, "ProjectName: %s" % smbproj
+
+	def client_hide_hosts(f, s):
+	    for a in s.get('fedAttr', [ ]):
+		if a.get('attribute', "") == 'hosts':
+		    print >>f, 'Hide: %s' % a.get('value', "")
+
+	client_service_out = {
+		'SMB': client_smb,
+		'tmcd': client_null,
+		'seer': client_null,
+		'userconfig': client_null,
+		'project_export': client_null,
+		'seer_master': client_seer_master,
+		'hide_hosts': client_hide_hosts,
+	    }
+
+	def client_seer_master_export(f, s):
+	    print >>f, "AddedNode: seer-master"
+
+	def client_seer_local_export(f, s):
+	    print >>f, "AddedNode: control"
+
+	client_export_service_out = {
+		'seer_master': client_seer_master_export,
+		'local_seer_control': client_seer_local_export,
+	    }
+
+	def server_port(f, s):
+	    p = urlparse(s.get('server', 'http://localhost'))
+	    print >>f, 'port: remote:%s:%s:%s' % (p.port, p.hostname, p.port) 
+
+	def server_null(f,s): pass
+
+	def server_seer(f, s):
+	    print >>f, 'seer: true'
+
+	server_service_out = {
+		'SMB': server_port,
+		'tmcd': server_port,
+		'userconfig': server_null,
+		'project_export': server_null,
+		'seer': server_seer,
+		'seer_master': server_port,
+		'hide_hosts': server_null,
+	    }
+	# XXX: end un hardcode this
+
+
+	seer_out = False
+	client_out = False
+	for e in [ e for e in topo.elements \
+		if isinstance(e, topdl.Computer) and e.get_attribute('portal')]:
+	    myname = e.name
+	    type = e.get_attribute('portal_type')
+
+	    info = conninfo_to_dict(myname, connInfo)
+
+	    if not info:
+		raise service_error(service_error.req,
+			"No connectivity info for %s" % myname)
+
+	    # Translate to physical name (ProtoGENI doesn't have DNS)
+	    physname = nodes.get(myname, { }).get('hostname', None)
+	    peer = info.get('peer', "")
+	    ldomain = self.domain
+	    ssh_port = info.get('ssh_port', 22)
+
+	    # Collect this for the client.conf file
+	    if 'masterexperiment' in info:
+		mproj, meid = info['masterexperiment'].split("/", 1)
+
+	    active = info.get('active', 'False')
+
+	    if type in ('control', 'both'):
+		testbed = e.get_attribute('testbed')
+		control_gw = myname
+
+	    cfn = "%s/%s.gw.conf" % (tmpdir, myname.lower())
+	    tunnelconfig = self.tunnel_config
+	    try:
+		f = open(cfn, "w")
+		if active == 'True':
+		    print >>f, "active: True"
+		    print >>f, "ssh_port: %s" % ssh_port
+		    if type in ('control', 'both'):
+			for s in [s for s in services \
+				if s.get('name', "") in self.imports]:
+			    server_service_out[s['name']](f, s)
+
+		if tunnelconfig:
+		    print >>f, "tunnelip: %s" % tunnelconfig
+		print >>f, "peer: %s" % peer.lower()
+		print >>f, "ssh_pubkey: /usr/local/federation/etc/%s" % \
+			pubkey_base
+		print >>f, "ssh_privkey: /usr/local/federation/etc/%s" % \
+			secretkey_base
+		f.close()
+	    except EnvironmentError, e:
+		raise service_error(service_error.internal,
+			"Can't write protal config %s: %s" % (cfn, e))
+
+	# Done with portals, write the client config file.
+	try:
+	    f = open("%s/client.conf" % tmpdir, "w")
+	    if control_gw:
+		print >>f, "ControlGateway: %s" % physname.lower()
+	    for s in services:
+		if s.get('name',"") in self.imports and \
+			s.get('visibility','') == 'import':
+		    client_service_out[s['name']](f, s)
+		if s.get('name', '') in self.exports and \
+			s.get('visibility', '') == 'export' and \
+			s['name'] in client_export_service_out:
+		    client_export_service_out[s['name']](f, s)
+	    # Seer uses this.
+	    if mproj and meid:
+		print >>f, "ExperimentID: %s/%s" % (mproj, meid)
+	    f.close()
+	except EnvironmentError, e:
+	    raise service_error(service_error.internal,
+		    "Cannot write client.conf: %s" %s)
+
+
+
+    def export_store_info(self, cf, nodes, ssh_port, connInfo):
+	"""
+	For the export requests in the connection info, install the peer names
+	at the experiment controller via SetValue calls.
+	"""
+
+	for c in connInfo:
+	    for p in [ p for p in c.get('parameter', []) \
+		    if p.get('type', '') == 'output']:
+
+		if p.get('name', '') == 'peer':
+		    k = p.get('key', None)
+		    surl = p.get('store', None)
+		    if surl and k and k.index('/') != -1:
+			if self.create_debug:
+			    req = { 'name': k, 'value': 'debug' }
+			    self.call_SetValue(surl, req, cf)
+			else:
+			    n = nodes.get(k[k.index('/')+1:], { })
+			    value = n.get('hostname', None)
+			    if value:
+				req = { 'name': k, 'value': value }
+				self.call_SetValue(surl, req, cf)
+			    else:
+				self.log.error("No hostname for %s" % \
+					k[k.index('/'):])
+		    else:
+			self.log.error("Bad export request: %s" % p)
+		elif p.get('name', '') == 'ssh_port':
+		    k = p.get('key', None)
+		    surl = p.get('store', None)
+		    if surl and k:
+			req = { 'name': k, 'value': ssh_port }
+			self.call_SetValue(surl, req, cf)
+		    else:
+			self.log.error("Bad export request: %s" % p)
+		else:
+
+		    self.log.error("Unknown export parameter: %s" % \
+			    p.get('name'))
+		    continue
+
+    def configure_nodes(self, segment_commands, topo, nodes, user, pubkey, secretkey, 
+	    stagingdir, tmpdir):
+
+	# These little functions/functors just make things more readable
+	class stage_file_type:
+	    def __init__(self, user, host, stagingdir):
+		self.user = user
+		self.host = host
+		self.stagingdir = stagingdir
+		self.scp = "/usr/bin/scp -i .ssh/id_rsa -o " + \
+			"'ForwardX11 no' -o 'StrictHostKeyChecking no' "
+
+	    def __call__(self, script, file, dest="."):
+		# If the file is a full pathname, do not use stagingdir
+		if file.find('/') == -1:
+		    file = "%s/%s" % (self.stagingdir, file)
+		print >>script, "%s %s@%s:%s %s" % \
+			(self.scp, self.user, self.host, file, dest)
+
+	def install_tar(script, loc, base):
+	    tar = "/bin/tar"
+	    mkdir="/bin/mkdir"
+
+	    print >>script, "%s -p %s" % (mkdir, loc)
+	    print >>script, "%s -C %s -xzf %s" % (tar, loc, base)
+
+	def install_rpm(script, base):
+	    rpm = "/bin/rpm"
+	    print >>script, "%s --install %s" % (rpm, base)
+
+	fed_dir = "/usr/local/federation"
+	fed_etc_dir = "%s/etc" % fed_dir
+	fed_bin_dir = "%s/bin" % fed_dir
+	fed_lib_dir = "%s/lib" % fed_dir
+
+	ifconfig = "/sbin/ifconfig"
+
+	stage_file = stage_file_type(user, self.staging_host, stagingdir)
+
+	for e in [ e for e in topo.elements if isinstance(e, topdl.Computer)]:
+	    vname = e.name
+	    node = nodes.get(vname, { })
+	    pname = node.get('hostname', None)
+	    if pname:
+		script = open("%s/%s.startup" %(tmpdir, pname), "w")
+		# Reset the interfaces to the ones in the topo file
+		for i in [ i for i in e.interface \
+			if not i.get_attribute('portal')]:
+		    pinf = node['interfaces'].get(i.name, None)
+		    addr = i.get_attribute('ip4_address') 
+		    netmask = i.get_attribute('ip4_netmask') or '255.255.255.0'
+		    if pinf and addr:
+			print >>script, \
+				"%s %s %s netmask %s"  % \
+				(ifconfig, pinf, addr, netmask)
+		    else:
+			self.log.error("Missing interface or address for %s" \
+				% i.name)
+		    
+		for l, f in self.federation_software:
+		    base = os.path.basename(f)
+		    stage_file(script, base)
+		    if l: install_tar(script, l, base)
+		    else: install_rpm(script, base)
+
+		for s in e.software:
+		    s_base = s.location.rpartition('/')[2]
+		    stage_file(script, s_base)
+		    if s.install: install_tar(script, s.install, s_base)
+		    else: install_rpm(script, s_base)
+
+		for f in ('hosts', pubkey, secretkey, 'client.conf', 
+			'userconf'):
+		    stage_file(script, f, fed_etc_dir)
+		if self.sshd:
+		    stage_file(script, self.sshd, fed_bin_dir)
+		if self.sshd_config:
+		    stage_file(script, self.sshd_config, fed_etc_dir)
+
+		# Look in tmpdir to get the names.  They've all been copied
+		# into the (remote) staging dir
+		if os.access("%s/%s.gw.conf" % (tmpdir, vname), os.R_OK):
+		    stage_file(script, "%s.gw.conf" % vname, fed_etc_dir)
+
+		# Hackery dackery dock: the ProtoGENI python is really ancient.
+		# A modern version (though packaged for Mandrake (remember
+		# Mandrake?  good times, good times)) should be in the
+		# federation_software list, but we need to move rename is for
+		# SEER.
+		print >>script, "rm /usr/bin/python"
+		print >>script, "ln /usr/bin/python2.4 /usr/bin/python"
+		# Back to less hacky stuff
+
+		# Start commands
+		if e.get_attribute('portal') and self.portal_startcommand:
+		    # Install portal software
+		    for l, f in self.portal_software:
+			base = os.path.basename(f)
+			stage_file(script, base)
+			if l: install_tar(script, l, base)
+			else: install_rpm(script, base)
+
+		    # Portals never have a user-specified start command
+		    print >>script, self.portal_startcommand
+		elif self.node_startcommand:
+		    # XXX: debug
+		    print >>script, "sudo perl -I%s %simport_key.pl /users/%s/.ssh/authorized_keys /root/.ssh/authorized_keys" % (fed_lib_dir, fed_bin_dir, user)
+		    # XXX: debug
+		    if e.get_attribute('startup'):
+			print >>script, "%s \\$USER '%s'" % \
+				(self.node_startcommand, e.get_attribute('startup'))
+		    else:
+			print >>script, self.node_startcommand
+		script.close()
+		if not segment_commands.scp_file("%s/%s.startup" % (tmpdir, pname), 
+			user, pname):
+		    self.log.error("Could not copy script to %s" % pname)
+	    else:
+		self.log.error("Unmapped node: %s" % vname)
+
+    def start_node(self, user, host, node, segment_commands):
+	# Place an identity on the node so that the copying can succeed
+	segment_commands.ssh_cmd(user, host, "scp .ssh/id_rsa %s:.ssh" % node)
+	segment_commands.ssh_cmd(user, node, 
+		"sudo /bin/sh ./%s.startup &" % node)
+
+    def start_nodes(self, user, host, nodes, segment_commands):
+	threads = [ ]
+	for n in nodes:
+	    t = Thread(target=self.start_node, args=(user, host, n, 
+		segment_commands))
+	    t.start()
+	    threads.append(t)
+
+	done = [not t.isAlive() for t in threads]
+	while not all(done):
+	    self.log.info("Waiting for threads %s" % done)
+	    time.sleep(10)
+	    done = [not t.isAlive() for t in threads]
+
+
+
+
+    def start_segment(self, segment_commands, aid, user, rspec, pubkey,
+	    secretkey, ename, stagingdir, tmpdir, certfile, certpw,
+	    export_certfile, topo, connInfo, services, timeout=0):
+	"""
+	Start a sub-experiment on a federant.
+
+	Get the current state, modify or create as appropriate, ship data
+	and configs and start the experiment.  There are small ordering
+	differences based on the initial state of the sub-experiment.
+	"""
+
+	def random_slicename(user):
+	    slicename = user
+	    for i in range(0,5):
+		slicename += random.choice(string.ascii_letters)
+	    return slicename
+
+	host = self.staging_host
+	if not os.access(certfile, os.R_OK):
+	    self.log.error("[start_segment]: Cannot read certfile: %s" % \
+		    certfile)
+	    return False
+	ctxt = fedd_ssl_context(my_cert=certfile, password=certpw)
+	# Local software dir
+	lsoftdir = "%s/software" % tmpdir
+
+	# Open up a temporary file to contain a script for setting up the
+	# filespace for the new experiment.
+	self.log.info("[start_segment]: creating script file")
+	try:
+	    sf, scriptname = tempfile.mkstemp()
+	    scriptfile = os.fdopen(sf, 'w')
+	except EnvironmentError:
+	    return False
+
+	scriptbase = os.path.basename(scriptname)
+
+	# Script the filesystem changes
+	print >>scriptfile, "/bin/rm -rf %s" % stagingdir
+	print >>scriptfile, 'mkdir -p %s' % stagingdir
+	print >>scriptfile, "rm -f %s" % scriptbase
+	scriptfile.close()
+
+	# Move the script to the remote machine
+	# XXX: could collide tempfile names on the remote host
+	if segment_commands.scp_file(scriptname, user, host, scriptbase):
+	    os.remove(scriptname)
+	else:
+	    return False
+
+	# Execute the script (and the script's last line deletes it)
+	if not segment_commands.ssh_cmd(user, host, "sh -x %s" % scriptbase):
+	    return False
+
+	try:
+	    gcred = segment_commands.pg_call(self.sa_url, 
+		    'GetCredential', {}, ctxt)
+	except self.ProtoGENIError, e:
+	    raise service_error(service_error.federant,
+		    "ProtoGENI: %s" % e)
+	# Find a slicename not in use
+	slicename = "fabereGpgL"
+	while True:
+	    slicename = random_slicename(user)
+	    try:
+		param = {
+			'credential': gcred, 
+			'hrn': slicename,
+			'type': 'Slice'
+			}
+		segment_commands.pg_call(self.sa_url, 'Resolve', param, ctxt)
+	    except segment_commands.ProtoGENIError, e:
+		print e
+		break
+
+	self.log.info("Creating %s" % slicename)
+	f = open("./rspec", "w")
+	print >>f, "%s" % rspec
+	f.close()
+	# Create the slice and allocate resources.  If any of this stuff fails,
+	# the allocations will time out on PG in short order, so we just raise
+	# the service_error.
+	try:
+	    param = {
+		    'credential': gcred, 
+		    'hrn': slicename,
+		    'type': 'Slice'
+		    }
+	    slice_cred = segment_commands.pg_call(self.sa_url, 'Register', param, ctxt)
+	    f = open("./slice_cred", "w")
+	    print >>f, slice_cred
+	    f.close()
+	    # Populate the ssh keys (let PG format them)
+	    param = {
+		    'credential': gcred, 
+		    }
+	    keys =  segment_commands.pg_call(self.sa_url, 'GetKeys', param, ctxt)
+	    # Grab and redeem a ticket
+	    param = {
+		    'credential': slice_cred, 
+		    'rspec': rspec,
+		    }
+	    ticket = segment_commands.pg_call(self.cm_url, 'GetTicket', param, ctxt)
+	    f = open("./ticket", "w")
+	    print >>f, ticket
+	    f.close()
+	    param = { 
+		    'credential': slice_cred, 
+		    'keys': keys,
+		    'ticket': ticket,
+		    }
+	    sliver_cred, manifest = segment_commands.pg_call(self.cm_url, 
+		    'RedeemTicket', param, ctxt)
+	    f = open("./sliver_cred", "w")
+	    print >>f, sliver_cred
+	    f.close()
+	    f = open("./manifest", "w")
+	    print >>f, manifest
+	    f.close()
+	    # start 'em up
+	    param = { 
+		    'credential': sliver_cred,
+		    }
+	    segment_commands.pg_call(self.cm_url, 'StartSliver', param, ctxt)
+	except segment_commands.ProtoGENIError, e:
+	    raise service_error(service_error.federant,
+		    "ProtoGENI: %s %s" % (e.code, e))
+
+	# With manifest in hand, we can export the portal node names.
+	if self.create_debug: nodes = self.fake_manifest(topo)
+	else: nodes = self.manifest_to_dict(manifest)
+
+	self.export_store_info(export_certfile, nodes, self.ssh_port,
+		connInfo)
+	self.generate_portal_configs(topo, pubkey, secretkey, tmpdir, 
+		ename, connInfo, services, nodes)
+
+	# Copy software to the staging machine (done after generation to copy
+	# those, too)
+	for d in (tmpdir, lsoftdir):
+	    if os.path.isdir(d):
+		for f in os.listdir(d):
+		    if not os.path.isdir("%s/%s" % (d, f)):
+			if not segment_commands.scp_file("%s/%s" % (d, f), 
+				user, host, "%s/%s" % (stagingdir, f)):
+			    self.log.error("Scp failed")
+			    return False
+
+
+	# Now we wait for the nodes to start on PG
+	status = 'notready'
+	try:
+	    while status == 'notready':
+		param = { 
+			'credential': slice_cred
+			}
+		r = segment_commands.pg_call(self.cm_url, 'SliceStatus', param, ctxt)
+		print r
+		status = r.get('status', 'notready')
+		if status == 'notready':
+		    time.sleep(30)
+	except segment_commands.ProtoGENIError, e:
+	    raise service_error(service_error.federant,
+		    "ProtoGENI: %s %s" % (e.code, e))
+
+	if status == 'failed':
+	    self.log.error('Sliver failed to start on ProtoGENI')
+	    try:
+		param = { 
+			'credential': slice_cred
+			}
+		segment_commands.pg_call(self.cm_url, 'DeleteSliver', param, ctxt)
+	    except segment_commands.ProtoGENIError, e:
+		raise service_error(service_error.federant,
+		    "ProtoGENI: %s" % e)
+	    return False
+	else:
+	    self.state_lock.acquire()
+	    self.allocation[aid]['slice_name'] = slicename
+	    self.allocation[aid]['slice_credential'] = slice_cred
+	    self.allocation[aid]['sliver_credential'] = sliver_cred
+	    self.allocation[aid]['manifest'] = manifest
+	    self.allocation[aid]['certfile'] = certfile
+	    self.allocation[aid]['certpw'] = certpw
+	    self.write_state()
+	    self.state_lock.release()
+
+	# Now we have configuration to do for ProtoGENI
+	self.configure_nodes(segment_commands, topo, nodes, user, pubkey, secretkey, 
+		stagingdir, tmpdir)
+
+	self.start_nodes(user, self.staging_host, 
+		[ n.get('hostname', None) for n in nodes.values()],
+		segment_commands)
+
+	# Everything has gone OK.
+	return True, dict([(k, n.get('hostname', None)) \
+		for k, n in nodes.items()])
 
     def generate_rspec(self, topo, softdir, connInfo):
@@ -457,12 +1138,12 @@
 		cpw, alloc_log)
 
-    def finalize_experiment(self, topo, starter, aid, alloc_id):
+    def finalize_experiment(self, topo, nodes, aid, alloc_id):
 	# Copy the assigned names into the return topology
 	rvtopo = topo.clone()
 	embedding = [ ]
-	for n in starter.node:
+	for k, n in nodes.items():
 	    embedding.append({ 
-		'toponame': n,
-		'physname': ["%s%s" %  (starter.node[n], self.domain)],
+		'toponame': k,
+		'physname': ["%s%s" %  (n, self.domain)],
 		})
 	# Grab the log (this is some anal locking, but better safe than
@@ -535,16 +1216,15 @@
 		cpw, alloc_log = self.initialize_experiment_info(attrs,
 			aid, certfile, tmpdir)
-	    # XXX: we really need to put the import and connection info
-	    # generation off longer.
 	    self.import_store_info(certfile, connInfo)
 	    rspec = self.generate_rspec(topo, "%s/%s/" \
 		    % (self.staging_dir, ename), connInfo)
 
-	    starter = self.start_segment(keyfile=ssh_key,
+	    segment_commands = protogeni_proxy(keyfile=ssh_key,
 		    debug=self.create_debug, log=alloc_log,
 		    ch_url = self.ch_url, sa_url=self.sa_url,
 		    cm_url=self.cm_url)
-	    rv = starter(self, aid, user, rspec, pubkey_base, secretkey_base,
-		    ename,
+	    rv, nodes = self.start_segment(segment_commands, aid, user, rspec,
+		    pubkey_base, 
+		    secretkey_base, ename,
 		    "%s/%s" % (self.staging_dir, ename), tmpdir, cf, cpw,
 		    certfile, topo, connInfo, services)
@@ -563,5 +1243,5 @@
 
 	if rv:
-	    return self.finalize_experiment(topo, starter, aid, req['allocID'])
+	    return self.finalize_experiment(topo, nodes, aid, req['allocID'])
 	elif err:
 	    raise service_error(service_error.federant,
@@ -569,4 +1249,32 @@
 	else:
 	    raise service_error(service_error.federant, "Swapin failed")
+
+    def stop_segment(self, segment_commands, user, stagingdir, slice_cred,
+	    certfile, certpw):
+	"""
+	Stop a sub experiment by calling swapexp on the federant
+	"""
+	host = self.staging_host
+	rv = False
+	try:
+	    # Clean out tar files: we've gone over quota in the past
+	    if stagingdir:
+		segment_commands.ssh_cmd(user, host, "rm -rf %s" % stagingdir)
+	    if slice_cred:
+		self.log.error('Removing Sliver on ProtoGENI')
+		ctxt = fedd_ssl_context(my_cert=certfile, password=certpw)
+		try:
+		    param = { 
+			    'credential': slice_cred
+			    }
+		    segment_commands.pg_call(self.cm_url, 'DeleteSlice',
+			    param, ctxt)
+		except segment_commands.ProtoGENIError, e:
+		    raise service_error(service_error.federant,
+			"ProtoGENI: %s" % e)
+	    return True
+	except self.ssh_cmd_timeout:
+	    rv = False
+	return rv
 
     def TerminateSegment(self, req, fid):
@@ -599,8 +1307,58 @@
 	    staging = None
 
-	stopper = self.stop_segment(keyfile=ssh_key, debug=self.create_debug,
-		ch_url = self.ch_url, sa_url=self.sa_url, cm_url=self.cm_url)
-	stopper(self, user, staging, slice_cred, cf, cpw)
+	segment_commands = protogeni_proxy(keyfile=ssh_key,
+		debug=self.create_debug, ch_url = self.ch_url,
+		sa_url=self.sa_url, cm_url=self.cm_url)
+	self.stop_segment(segment_commands, user, staging, slice_cred, cf, cpw)
 	return { 'allocID': req['allocID'] }
+
+    def renew_segment(self, segment_commands, name, scred, interval, 
+	    certfile, certpw):
+	ctxt = fedd_ssl_context(my_cert=certfile, password=certpw)
+	try:
+	    expiration = time.strftime("%Y%m%dT%H:%M:%S",
+		    time.gmtime(time.time() + interval))
+	    cred = segment_commands.pg_call(self.sa_url, 'GetCredential', {}, ctxt)
+
+	    param = {
+		    'credential': scred,
+		    'expiration': expiration
+		    }
+	    r = segment_commands.pg_call(self.sa_url, 'RenewSlice', param, ctxt)
+	    param = {
+		    'credential': cred,
+		    'hrn': name,
+		    'type': 'Slice',
+		    }
+	    slice = segment_commands.pg_call(self.sa_url, 'Resolve', param, ctxt)
+	    uuid = slice.get('uuid', None)
+	    if uuid == None:
+		sys.exit('No uuid for %s' % slicename)
+
+	    print 'Calling GetCredential (uuid)'
+	    param = {
+		    'credential': cred,
+		    'uuid': uuid,
+		    'type': 'Slice',
+		    }
+	    new_scred = segment_commands.pg_call(self.sa_url, 'GetCredential', param, ctxt)
+	    f = open('./new_slice_cred', 'w')
+	    print >>f, new_scred
+	    f.close()
+
+	except segment_commands.ProtoGENIError, e:
+	    self.log.error("Failed to extend slice %s: %s" % (name, e))
+	    return None
+	try:
+	    print 'Calling RenewSlice (CM)'
+	    param = {
+		    'credential': new_scred,
+		    }
+	    r = segment_commands.pg_call(self.cm_url, 'RenewSlice', param, ctxt)
+	except segment_commands.ProtoGENIError, e:
+	    self.log.warn("Failed to renew sliver for %s: %s" % (name, e))
+
+	return new_scred
+    
 
     def RenewSlices(self):
@@ -628,9 +1386,10 @@
 	    # There's a ProtoGENI slice associated with the segment; renew it.
 	    if name and scred:
-		renewer = self.renew_segment(log=self.log, 
+		segment_commands = protogeni_proxy(log=self.log, 
 			debug=self.create_debug, keyfile=ssh_key,
 			cm_url = self.cm_url, sa_url = self.sa_url,
 			ch_url = self.ch_url)
-		new_scred = renewer(name, scred, self.renewal_interval, cf, cpw)
+		new_scred = self.renew_segment(segment_commands, name, scred, 
+			self.renewal_interval, cf, cpw)
 		if new_scred:
 		    self.log.info("Slice %s renewed until %s GMT" % \
