Index: fedd/federation/experiment_control.py
===================================================================
--- fedd/federation/experiment_control.py	(revision db6b092dbd30839cdaeb24ac51df1e525a79befa)
+++ fedd/federation/experiment_control.py	(revision cc8d8e971b7119cdebf6c0a7ae1d1da299531ce6)
@@ -196,4 +196,5 @@
     call_RequestAccess = service_caller('RequestAccess')
     call_ReleaseAccess = service_caller('ReleaseAccess')
+    call_StartSegment = service_caller('StartSegment')
     call_Ns2Split = service_caller('Ns2Split')
 
@@ -230,4 +231,7 @@
         self.trusted_certs = config.get("experiment_control", "trusted_certs") \
                 or config.get("globals", "trusted_certs")
+
+	# XXX:
+	self.repodir = '/usr/local/etc/fedd/repo'
 
         self.exp_stem = "fed-stem"
@@ -403,4 +407,32 @@
         State format is a simple pickling of the state dictionary.
         """
+	
+	def get_experiment_id(state):
+	    """
+	    Pull the fedid experimentID out of the saved state.  This is kind
+	    of a gross walk through the dict.
+	    """
+
+	    if state.has_key('experimentID'):
+		for e in state['experimentID']:
+		    if e.has_key('fedid'):
+			return e['fedid']
+		else:
+		    return None
+	    else:
+		return None
+
+	def get_alloc_ids(state):
+	    """
+	    Pull the fedids of the identifiers of each allocation from the
+	    state.  Again, a dict dive that's best isolated.
+	    """
+
+	    return [ f['allocID']['fedid'] 
+		    for f in state.get('federant',[]) \
+			if f.has_key('allocID') and \
+			    f['allocID'].has_key('fedid')]
+
+
         try:
             f = open(self.state_filename, "r")
@@ -415,16 +447,22 @@
                     "Unpickling failed: %s") % e)
         
-        for k in self.state.keys():
+        for s in self.state.values():
             try:
-                # This list should only have one element in it, but phrasing it
-                # as a for loop doesn't cost much, really.  We have to find the
-                # fedid elements anyway.
-                for eid in [ f['fedid'] \
-                        for f in self.state[k]['experimentID']\
-                            if f.has_key('fedid') ]:
-                    self.auth.set_attribute(self.state[k]['owner'], eid)
+
+		eid = get_experiment_id(s)
+		if eid : 
+		    # Give the owner rights to the experiment
+                    self.auth.set_attribute(s['owner'], eid)
+		    # And holders of the eid as well
+                    self.auth.set_attribute(eid, eid)
                     # allow overrides to control experiments as well
                     for o in self.overrides:
                         self.auth.set_attribute(o, eid)
+		    # Set permissions to allow reading of the software repo, if
+		    # any, as well.
+		    for a in get_alloc_ids(s):
+			self.auth.set_attribute(a, 'repo/%s' % eid)
+		else:
+		    raise KeyError("No experiment id")
             except KeyError, e:
                 self.log.warning("[read_state]: State ownership or identity " +\
@@ -970,6 +1008,6 @@
             nodes = [ n['vname'] for n in topo['node'] ]
             topo_lans = topo['lan']
-        except KeyError:
-            raise service_error(service_error.internal, "Bad topology")
+        except KeyError, e:
+	    raise service_error(service_error.internal, "Bad topology: %s" %e)
 
         lans = { }
@@ -2103,5 +2141,146 @@
 
         return rv
+
+    class new_start_segment:
+	def __init__(self, debug=False, log=None, cert_file=None,
+		cert_pwd=None, trusted_certs=None, caller=None):
+	    self.log = log
+	    self.debug = debug
+	    self.cert_file = cert_file
+	    self.cert_pwd = cert_pwd
+	    self.trusted_certs = None
+	    self.caller = caller
+
+	def __call__(self, uri, aid, topo):
+	    req = {
+		    'allocID': { 'fedid' : aid }, 
+		    'segmentdescription': { 
+			'topdldescription': topo.to_dict(),
+		    },
+		}
+
+	    print "calling %s"  % uri
+	    r = self.caller(uri, req, self.cert_file, self.cert_pwd,
+		    self.trusted_certs)
+	    print r
+	    return True
+
+
     
+
+    def new_allocate_resources(self, allocated, master, eid, expid, expcert, 
+            tbparams, topo, tmpdir, alloc_log=None):
+        started = { }           # Testbeds where a sub-experiment started 
+                                # successfully
+
+        # XXX
+        fail_soft = False
+
+        log = alloc_log or self.log
+
+        thread_pool = self.thread_pool(self.nthreads)
+        threads = [ ]
+
+        for tb in [ k for k in allocated.keys() if k != master]:
+            # Create and start a thread to start the segment, and save it to
+            # get the return value later
+            thread_pool.wait_for_slot()
+	    uri = self.tbmap.get(tb, None)
+	    if not uri:
+		raise service_error(service_error.internal, 
+			"Unknown testbed %s !?" % tb)
+
+	    if tbparams[tb].has_key('allocID') and \
+		    tbparams[tb]['allocID'].has_key('fedid'):
+		aid = tbparams[tb]['allocID']['fedid']
+	    else:
+		raise service_error(service_error.internal, 
+			"No alloc id for testbed %s !?" % tb)
+
+            t  = self.pooled_thread(\
+                    target=self.new_start_segment(log=log, debug=self.debug,
+			cert_file=self.cert_file, cert_pwd=self.cert_pwd,
+			trusted_certs=self.trusted_certs,
+			caller=self.call_StartSegment), 
+                    args=(uri, aid, topo[tb]), name=tb,
+                    pdata=thread_pool, trace_file=self.trace_file)
+            threads.append(t)
+            t.start()
+
+        # Wait until all finish 
+        thread_pool.wait_for_all_done()
+
+        # If none failed, start the master
+        failed = [ t.getName() for t in threads if not t.rv ]
+
+        if len(failed) == 0:
+	    uri = self.tbmap.get(master, None)
+	    if not uri:
+		raise service_error(service_error.internal, 
+			"Unknown testbed %s !?" % master)
+
+	    if tbparams[master].has_key('allocID') and \
+		    tbparams[master]['allocID'].has_key('fedid'):
+		aid = tbparams[master]['allocID']['fedid']
+	    else:
+		raise service_error(service_error.internal, 
+		    "No alloc id for testbed %s !?" % master)
+            starter = self.new_start_segment(log=log, debug=self.debug,
+		    cert_file=self.cert_file, cert_pwd=self.cert_pwd,
+		    trusted_certs=self.trusted_certs,
+		    caller=self.call_StartSegment)
+	    if not starter(uri, aid, topo[master]):
+                failed.append(master)
+
+        succeeded = [tb for tb in allocated.keys() if tb not in failed]
+        # If one failed clean up, unless fail_soft is set
+        if failed and False:
+            if not fail_soft:
+                thread_pool.clear()
+                for tb in succeeded:
+                    # Create and start a thread to stop the segment
+                    thread_pool.wait_for_slot()
+                    t  = self.pooled_thread(\
+                            target=self.stop_segment(log=log,
+                                keyfile=self.ssh_privkey_file,
+                                debug=self.debug), 
+                            args=(tb, eid, tbparams), name=tb,
+                            pdata=thread_pool, trace_file=self.trace_file)
+                    t.start()
+                # Wait until all finish
+                thread_pool.wait_for_all_done()
+
+                # release the allocations
+                for tb in tbparams.keys():
+                    self.release_access(tb, tbparams[tb]['allocID'])
+                # Remove the placeholder
+                self.state_lock.acquire()
+                self.state[eid]['experimentStatus'] = 'failed'
+                if self.state_filename: self.write_state()
+                self.state_lock.release()
+
+                log.error("Swap in failed on %s" % ",".join(failed))
+                return
+        else:
+            log.info("[start_segment]: Experiment %s active" % eid)
+
+        log.debug("[start_experiment]: removing %s" % tmpdir)
+
+        # Walk up tmpdir, deleting as we go
+        for path, dirs, files in os.walk(tmpdir, topdown=False):
+            for f in files:
+                os.remove(os.path.join(path, f))
+            for d in dirs:
+                os.rmdir(os.path.join(path, d))
+        os.rmdir(tmpdir)
+
+        # Insert the experiment into our state and update the disk copy
+        self.state_lock.acquire()
+        self.state[expid]['experimentStatus'] = 'active'
+        self.state[eid] = self.state[expid]
+        if self.state_filename: self.write_state()
+        self.state_lock.release()
+        return
+
 
     def new_create_experiment(self, req, fid):
@@ -2274,258 +2453,208 @@
 
             allocated = { }         # Testbeds we can access
-# XXX here's where we're working
-            def out_topo(filename, t):
-                try:
-                    f = open("/tmp/%s" % filename, "w")
-                    print >> f, "%s" % \
-                            topdl.topology_to_xml(t, top="experiment")
-                    f.close()
-                except IOError, e:
-                    raise service_error(service_error.internal, "Can't open file")
-
-            try:
-
-                top = topdl.topology_from_xml(file=split_data, top="experiment")
-                subs = sorted(top.substrates, 
-                        cmp=lambda x,y: cmp(len(x.interfaces), len(y.interfaces)),
-                        reverse=True)
-                ips = ip_allocator(int(ip_addr("10.0.0.0")), 2 **24)
-                for s in subs:
-                    a = ips.allocate(len(s.interfaces)+2)
-                    if a :
-                        base, num = a
-                        if num < len(s.interfaces) +2 : 
-                            raise service_error(service_error.internal,
-                                    "Allocator returned wrong number of IPs??")
-                    else:
-                        raise service_error(service_error.req, 
-                                "Cannot allocate IP addresses")
-
-                    base += 1
-                    for i in s.interfaces:
-                        i.attribute.append(
-                                topdl.Attribute('ip4_address', 
-                                    "%s" % ip_addr(base)))
-                        base += 1
-
-                testbeds = set([ a.value for e in top.elements \
-                        for a in e.attribute \
-                            if a.attribute == 'testbed'] )
-                topo ={ }
-                for tb in testbeds:
-                    self.get_access(tb, None, user, tbparams, master,
-                            export_project, access_user)
-                    topo[tb] = top.clone()
-                    to_delete = [ ]
-                    for e in topo[tb].elements:
-                        etb = e.get_attribute('testbed')
-                        if etb and etb != tb:
-                            for i in e.interface:
-                                for s in i.subs:
-                                    try:
-                                        s.interfaces.remove(i)
-                                    except ValueError:
-                                        raise service_error(service_error.internal,
-                                                "Can't remove interface??")
-                            to_delete.append(e)
-                    for e in to_delete:
-                        topo[tb].elements.remove(e)
-                    topo[tb].make_indices()
-
-
-
-                for s in top.substrates:
-                    tests = { }
-                    for i in s.interfaces:
-                        e = i.element
-                        tb = e.get_attribute('testbed')
-                        if tb and not tests.has_key(tb):
-                            for i in e.interface:
-                                if s in i.subs:
-                                    tests[tb]= \
-                                            i.get_attribute('ip4_address')
-                    if len(tests) < 2:
-                        continue
-
-                    # More than one testbed is on this substrate.  Insert
-                    # some gateways into the subtopologies.
-
-                    for st in tests.keys():
-                        for dt in [ t for t in tests.keys() if t != st]:
-                            myname =  "%stunnel" % dt
-                            desthost  =  "%stunnel" % st
-                            sproject = tbparams[st].get('project', 'project')
-                            dproject = tbparams[dt].get('project', 'project')
-                            sdomain = ".%s.%s%s" % (eid, sproject,
-                                    tbparams[st].get('domain', ".example.com"))
-                            ddomain = ".%s.%s%s" % (eid, dproject,
-                                    tbparams[dt].get('domain', ".example.com"))
-                            boss = tbparams[master].get('boss', "boss")
-                            fs = tbparams[master].get('fs', "fs")
-                            event_server = "%s%s" % \
-                                    (tbparams[st].get('eventserver', "event_server"),
-                                            tbparams[dt].get('domain', "example.com"))
-                            remote_event_server = "%s%s" % \
-                                    (tbparams[dt].get('eventserver', "event_server"),
-                                            tbparams[dt].get('domain', "example.com"))
-                            seer_control = "%s%s" % \
-                                    (tbparams[st].get('control', "control"), sdomain)
-                            local_key_dir = "/proj/%s/exp/%s/tmp" % ( sproject, eid)
-                            remote_conf_dir = "/proj/%s/exp/%s/tmp" % ( dproject, eid)
-                            conf_file = "%s%s.gw.conf" % (myname, sdomain)
-                            remote_conf_file = "%s%s.gw.conf" % (desthost, ddomain)
-                            # translate to lower case so the `hostname` hack for specifying
-                            # configuration files works.
-                            conf_file = conf_file.lower();
-                            remote_conf_file = remote_conf_file.lower();
-                            active = ("%s" % (st == master))
-                            portal = topdl.Computer(**{
-                                    'name': "%stunnel" % dt,
-                                    'attribute' : [{
-                                        'attribute': n,
-                                        'value': v,
-                                        } for n, v in (\
-                                                ('gateway', 'true'),
-                                                ('boss', boss),
-                                                ('fs', fs),
-                                                ('event_server', event_server),
-                                                ('remote_event_server', remote_event_server),
-                                                ('seer_control', seer_control),
-                                                ('local_key_dir', local_key_dir),
-                                                ('remote_conf_dir', remote_conf_dir),
-                                                ('conf_file', conf_file),
-                                                ('remote_conf_file', remote_conf_file),
-                                                ('remote_script_dir', "/usr/local/federation/bin"),
-                                                ('local_script_dir', "/usr/local/federation/bin"),
-                                                )],
-                                    'interface': [{
-                                        'substrate': s.name,
-                                        'attribute': [ { 
-                                            'attribute': 'ip4_addreess', 
-                                            'value': tests[dt],
-                                            }, ],
-                                        }, ],
-                                    })
-                            topo[st].elements.append(portal)
-                # Connect the gateway nodes into the topologies and clear out
-                # substrates that are not in the topologies
-                for tb in testbeds:
-                    topo[tb].incorporate_elements()
-                    topo[tb].substrates = \
-                            [s for s in topo[tb].substrates \
-                                if len(s.interfaces) >0]
-
-                softdir ="%s/software" % tmpdir
-                softmap = { }
-                os.mkdir(softdir)
-                pkgs = set([fedkit, gatewaykit])
-                pkgs.update([x.location for e in top.elements \
-                        for x in e.software])
-                for pkg in pkgs:
-                    loc = pkg
-
-                    scheme, host, path = urlparse(loc)[0:3]
-                    dest = os.path.basename(path)
-                    if not scheme:
-                        if not loc.startswith('/'):
-                            loc = "/%s" % loc
-                        loc = "file://%s" %loc
-                    try:
-                        u = urlopen(loc)
-                    except Exception, e:
-                        raise service_error(service_error.req, 
-                                "Cannot open %s: %s" % (loc, e))
-                    try:
-                        f = open("%s/%s" % (softdir, dest) , "w")
-                        data = u.read(4096)
-                        while data:
-                            f.write(data)
-                            data = u.read(4096)
-                        f.close()
-                        u.close()
-                    except Exception, e:
-                        raise service_error(service_error.internal,
-                                "Could not copy %s: %s" % (loc, e))
-                    path = re.sub("/tmp", "", softdir)
-                    # XXX
-                    softmap[pkg] = \
-                            "https://users.isi.deterlab.net:23232/%s/%s" %\
-                            ( path, dest)
-
-                # Convert the software locations in the segments into the local
-                # copies on this host
-                for soft in [ s for tb in topo.values() \
-                        for e in tb.elements \
-                            for s in e.software ]:
-                    if softmap.has_key(soft.location):
-                        soft.location = softmap[soft.location]
-                for tb in testbeds:
-                    out_topo("%s.xml" %tb, topo[tb])
-
-                vtopo = topdl.topology_to_vtopo(top)
-                vis = self.genviz(vtopo)
-
-            except Exception, e:
-                traceback.print_exc()
-                raise service_error(service_error.internal, "%s"  % e)
-
-
-
-            # Build the testbed topologies:
-
-
-            if True:
-                raise service_error(service_error.internal, "Developing")
-
-# XXX old code
-            # Objects to parse the splitter output (defined above)
-            parse_current_testbed = self.current_testbed(eid, tmpdir,
-                    self.fedkit, self.gatewaykit)
-            parse_allbeds = self.allbeds(self.get_access)
-            parse_gateways = self.gateways(eid, master, tmpdir,
-                    gw_pubkey_base, gw_secretkey_base, self.copy_file,
-                    self.fedkit)
-            parse_vtopo = self.shunt_to_string("^#\s+Begin\s+Vtopo",
-                        "^#\s+End\s+Vtopo")
-            parse_hostnames = self.shunt_to_file("^#\s+Begin\s+hostnames",
-                        "^#\s+End\s+hostnames", tmpdir + "/hosts")
-            parse_tarfiles = self.shunt_to_list("^#\s+Begin\s+tarfiles",
-                    "^#\s+End\s+tarfiles")
-            parse_rpms = self.shunt_to_list("^#\s+Begin\s+rpms",
-                    "^#\s+End\s+rpms")
-
-            # Working on the split data
-            for line in split_data:
-                line = line.rstrip()
-                if parse_current_testbed(line, master, allocated, tbparams):
-                    continue
-                elif parse_allbeds(line, user, tbparams, master, export_project,
-                        access_user):
-                    continue
-                elif parse_gateways(line, allocated, tbparams):
-                    continue
-                elif parse_vtopo(line):
-                    continue
-                elif parse_hostnames(line):
-                    continue
-                elif parse_tarfiles(line):
-                    continue
-                elif parse_rpms(line):
-                    continue
-                else:
-                    raise service_error(service_error.internal, 
-                            "Bad tcl parse? %s" % line)
-            # Virtual topology and visualization
-            vtopo = self.gentopo(parse_vtopo.str)
-            if not vtopo:
-                raise service_error(service_error.internal, 
-                        "Failed to generate virtual topology")
-
-            vis = self.genviz(vtopo)
-            if not vis:
-                raise service_error(service_error.internal, 
-                        "Failed to generate visualization")
-
-            
+	    # Allocate IP addresses: The allocator is a buddy system memory
+	    # allocator.  Allocate from the largest substrate to the
+	    # smallest to make the packing more likely to work - i.e.
+	    # avoiding internal fragmentation.
+	    top = topdl.topology_from_xml(file=split_data, top="experiment")
+	    subs = sorted(top.substrates, 
+		    cmp=lambda x,y: cmp(len(x.interfaces), 
+			len(y.interfaces)),
+		    reverse=True)
+	    ips = ip_allocator(int(ip_addr("10.0.0.0")), 2 **24)
+	    for s in subs:
+		a = ips.allocate(len(s.interfaces)+2)
+		if a :
+		    base, num = a
+		    if num < len(s.interfaces) +2 : 
+			raise service_error(service_error.internal,
+				"Allocator returned wrong number of IPs??")
+		else:
+		    raise service_error(service_error.req, 
+			    "Cannot allocate IP addresses")
+
+		base += 1
+		for i in s.interfaces:
+		    i.attribute.append(
+			    topdl.Attribute('ip4_address', 
+				"%s" % ip_addr(base)))
+		    base += 1
+
+	    # Find the testbeds to look up
+	    testbeds = set([ a.value for e in top.elements \
+		    for a in e.attribute \
+			if a.attribute == 'testbed'] )
+
+	    # Make per testbed topologies.  Copy the main topo and remove
+	    # interfaces and nodes that don't live in the testbed.
+	    topo ={ }
+	    for tb in testbeds:
+		self.get_access(tb, None, user, tbparams, master,
+			export_project, access_user)
+		allocated[tb] = 1
+		topo[tb] = top.clone()
+		to_delete = [ ]
+		for e in topo[tb].elements:
+		    etb = e.get_attribute('testbed')
+		    if etb and etb != tb:
+			for i in e.interface:
+			    for s in i.subs:
+				try:
+				    s.interfaces.remove(i)
+				except ValueError:
+				    raise service_error(service_error.internal,
+					    "Can't remove interface??")
+			to_delete.append(e)
+		for e in to_delete:
+		    topo[tb].elements.remove(e)
+		topo[tb].make_indices()
+
+
+
+	    # Now, for each substrate in the main topology, find those that
+	    # have nodes on more than one testbed.  Insert portal nodes
+	    # into the copies of those substrates on the sub topologies.
+	    for s in top.substrates:
+		tests = { }
+		for i in s.interfaces:
+		    e = i.element
+		    tb = e.get_attribute('testbed')
+		    if tb and not tests.has_key(tb):
+			for i in e.interface:
+			    if s in i.subs:
+				tests[tb]= \
+					i.get_attribute('ip4_address')
+		if len(tests) < 2:
+		    continue
+
+		# More than one testbed is on this substrate.  Insert
+		# some portals into the subtopologies.
+
+		for st in tests.keys():
+		    for dt in [ t for t in tests.keys() if t != st]:
+			myname =  "%stunnel" % dt
+			desthost  =  "%stunnel" % st
+			sproject = tbparams[st].get('project', 'project')
+			dproject = tbparams[dt].get('project', 'project')
+			sdomain = ".%s.%s%s" % (eid, sproject,
+				tbparams[st].get('domain', ".example.com"))
+			ddomain = ".%s.%s%s" % (eid, dproject,
+				tbparams[dt].get('domain', ".example.com"))
+			boss = tbparams[master].get('boss', "boss")
+			fs = tbparams[master].get('fs', "fs")
+			event_server = "%s%s" % \
+				(tbparams[st].get('eventserver', "event_server"),
+					tbparams[dt].get('domain', "example.com"))
+			remote_event_server = "%s%s" % \
+				(tbparams[dt].get('eventserver', "event_server"),
+					tbparams[dt].get('domain', "example.com"))
+			seer_control = "%s%s" % \
+				(tbparams[st].get('control', "control"), sdomain)
+			local_key_dir = "/proj/%s/exp/%s/tmp" % ( sproject, eid)
+			remote_conf_dir = "/proj/%s/exp/%s/tmp" % ( dproject, eid)
+			conf_file = "%s%s.gw.conf" % (myname, sdomain)
+			remote_conf_file = "%s%s.gw.conf" % (desthost, ddomain)
+			# translate to lower case so the `hostname` hack for specifying
+			# configuration files works.
+			conf_file = conf_file.lower();
+			remote_conf_file = remote_conf_file.lower();
+			active = ("%s" % (st == master))
+			portal = topdl.Computer(**{
+				'name': "%stunnel" % dt,
+				'attribute' : [{
+				    'attribute': n,
+				    'value': v,
+				    } for n, v in (\
+					    ('gateway', 'true'),
+					    ('boss', boss),
+					    ('fs', fs),
+					    ('event_server', event_server),
+					    ('remote_event_server', remote_event_server),
+					    ('seer_control', seer_control),
+					    ('local_key_dir', local_key_dir),
+					    ('remote_conf_dir', remote_conf_dir),
+					    ('conf_file', conf_file),
+					    ('remote_conf_file', remote_conf_file),
+					    ('remote_script_dir', "/usr/local/federation/bin"),
+					    ('local_script_dir', "/usr/local/federation/bin"),
+					    )],
+				'interface': [{
+				    'substrate': s.name,
+				    'attribute': [ { 
+					'attribute': 'ip4_addreess', 
+					'value': tests[dt],
+					}, ],
+				    }, ],
+				})
+			topo[st].elements.append(portal)
+	    # Connect the gateway nodes into the topologies and clear out
+	    # substrates that are not in the topologies
+	    for tb in testbeds:
+		topo[tb].incorporate_elements()
+		topo[tb].substrates = \
+			[s for s in topo[tb].substrates \
+			    if len(s.interfaces) >0]
+
+	    # Copy the rpms and tarfiles to a distribution directory from
+	    # which the federants can retrieve them
+	    linkpath = "%s/software" %  expid
+	    softdir ="%s/%s" % ( self.repodir, linkpath)
+	    softmap = { }
+	    pkgs = set([ t for l in [self.fedkit, self.gatewaykit] \
+		    for p, t in l ])
+	    pkgs.update([x.location for e in top.elements \
+		    for x in e.software])
+	    try:
+		os.makedirs(softdir)
+	    except IOError, e:
+		raise service_error(
+			"Cannot create software directory: %s" % e)
+	    for pkg in pkgs:
+		loc = pkg
+
+		scheme, host, path = urlparse(loc)[0:3]
+		dest = os.path.basename(path)
+		if not scheme:
+		    if not loc.startswith('/'):
+			loc = "/%s" % loc
+		    loc = "file://%s" %loc
+		try:
+		    u = urlopen(loc)
+		except Exception, e:
+		    raise service_error(service_error.req, 
+			    "Cannot open %s: %s" % (loc, e))
+		try:
+		    f = open("%s/%s" % (softdir, dest) , "w")
+		    data = u.read(4096)
+		    while data:
+			f.write(data)
+			data = u.read(4096)
+		    f.close()
+		    u.close()
+		except Exception, e:
+		    raise service_error(service_error.internal,
+			    "Could not copy %s: %s" % (loc, e))
+		path = re.sub("/tmp", "", linkpath)
+		# XXX
+		softmap[pkg] = \
+			"https://users.isi.deterlab.net:23232/%s/%s" %\
+			( path, dest)
+
+		# Allow the individual testbeds to access the software.
+		for tb in tbparams.keys():
+		    self.auth.set_attribute(tbparams[tb]['allocID']['fedid'], 
+			    "%s/%s" % ( path, dest))
+
+	    # Convert the software locations in the segments into the local
+	    # copies on this host
+	    for soft in [ s for tb in topo.values() \
+		    for e in tb.elements \
+			for s in e.software ]:
+		if softmap.has_key(soft.location):
+		    soft.location = softmap[soft.location]
+
+	    vtopo = topdl.topology_to_vtopo(top)
+	    vis = self.genviz(vtopo)
+
             # save federant information
             for k in allocated.keys():
@@ -2543,43 +2672,7 @@
                     [ tbparams[tb]['federant'] for tb in tbparams.keys() \
                         if tbparams[tb].has_key('federant') ]
-            if self.state_filename: self.write_state()
+            if self.state_filename: 
+		self.write_state()
             self.state_lock.release()
-
-            # Copy tarfiles and rpms needed at remote sites into a staging area
-            try:
-                if self.fedkit:
-                    for t in self.fedkit:
-                        parse_tarfiles.list.append(t[1])
-                if self.gatewaykit:
-                    for t in self.gatewaykit:
-                        parse_tarfiles.list.append(t[1])
-                for t in parse_tarfiles.list:
-                    if not os.path.exists("%s/tarfiles" % tmpdir):
-                        os.mkdir("%s/tarfiles" % tmpdir)
-                    self.copy_file(t, "%s/tarfiles/%s" % \
-                            (tmpdir, os.path.basename(t)))
-                for r in parse_rpms.list:
-                    if not os.path.exists("%s/rpms" % tmpdir):
-                        os.mkdir("%s/rpms" % tmpdir)
-                    self.copy_file(r, "%s/rpms/%s" % \
-                            (tmpdir, os.path.basename(r)))
-                # A null experiment file in case we need to create a remote
-                # experiment from scratch
-                f = open("%s/null.tcl" % tmpdir, "w")
-                print >>f, """
-set ns [new Simulator]
-source tb_compat.tcl
-
-set a [$ns node]
-
-$ns rtproto Session
-$ns run 
-"""
-                f.close()
-
-            except IOError, e:
-                raise service_error(service_error.internal, 
-                        "Cannot stage tarfile/rpm: %s" % e.strerror)
-
         except service_error, e:
             # If something goes wrong in the parse (usually an access error)
@@ -2587,4 +2680,5 @@
             # exceptions.  Failing at this point returns a fault to the remote
             # caller. 
+
             self.state_lock.acquire()
             del self.state[eid]
@@ -2616,7 +2710,7 @@
         
         # Start a thread to do the resource allocation
-        t  = Thread(target=self.allocate_resources,
+        t  = Thread(target=self.new_allocate_resources,
                 args=(allocated, master, eid, expid, expcert, tbparams, 
-                    tmpdir, alloc_log),
+                    topo, tmpdir, alloc_log),
                 name=eid)
         t.start()
