Index: fedd/federation/access.py
===================================================================
--- fedd/federation/access.py	(revision f9ef40b3f313c74ac1d4f6ec50841ede5fb89cda)
+++ fedd/federation/access.py	(revision f5ae004d9a2bc2b10b9c109a328f6c716fe84dda)
@@ -9,4 +9,6 @@
 
 from threading import *
+import subprocess
+import time
 
 from util import *
@@ -63,4 +65,5 @@
         self.eventserver = config.get("access", "eventserver")
 	self.certdir = config.get("access","certdir")
+	self.ssh_privkey_file = config.get("access","ssh_privkey_file")
 
         self.attrs = { }
@@ -616,4 +619,7 @@
                 else: self.projects[pname] = 1
                 self.allocation[aid]['project'] = pname
+	    else:
+		# sproject is a static project associated with this allocation.
+		self.allocation[aid]['sproject'] = pname
 
             if ap.has_key('resources'):
@@ -638,4 +644,6 @@
                 for u in ap['project']['user']:
                     uname = u['userID']['localname']
+		    if u['role'] == 'experimentCreation':
+			self.allocation[aid]['user'] = uname
                     for k in [ k['sshPubkey'] for k in u['access'] \
                             if k.has_key('sshPubkey') ]:
@@ -789,15 +797,15 @@
 
 
-    class emulab_segment:
+    class proxy_emulab_segment:
 	class ssh_cmd_timeout(RuntimeError): pass
 
         def __init__(self, log=None, keyfile=None, debug=False):
             self.log = log or logging.getLogger(\
-                    'fedd.experiment_control.emulab_segment')
+                    'fedd.access.proxy_emulab_segment')
             self.ssh_privkey_file = keyfile
             self.debug = debug
             self.ssh_exec="/usr/bin/ssh"
             self.scp_exec = "/usr/bin/scp"
-            self.ssh_cmd_timeout = emulab_segment.ssh_cmd_timeout
+            self.ssh_cmd_timeout = access.proxy_emulab_segment.ssh_cmd_timeout
 
         def scp_file(self, file, user, host, dest=""):
@@ -822,6 +830,6 @@
             self.log.debug("[scp_file]: %s" % " ".join(scp_cmd))
             if not self.debug:
-                rv = call(scp_cmd, stdout=dnull, stderr=dnull, close_fds=True,
-                        close_fds=True)
+                rv = subprocess.call(scp_cmd, stdout=dnull, 
+			stderr=dnull, close_fds=True, close_fds=True)
 
             return rv == 0
@@ -848,9 +856,8 @@
             if not self.debug:
                 if dnull:
-                    sub = Popen(sh_str, shell=True, stdout=dnull, stderr=dnull,
-                            close_fds=True)
+                    sub = subprocess.Popen(sh_str, shell=True, stdout=dnull,
+			    stderr=dnull, close_fds=True)
                 else:
-                    sub = Popen(sh_str, shell=True,
-                            close_fds=True)
+                    sub = subprocess.Popen(sh_str, shell=True, close_fds=True)
                 if timeout:
                     i = 0
@@ -876,43 +883,19 @@
                     return True
 
-    class start_segment(emulab_segment):
+    class start_segment(proxy_emulab_segment):
         def __init__(self, log=None, keyfile=None, debug=False):
-            experiment_control_local.emulab_segment.__init__(self,
-                    log=log, keyfile=keyfile, debug=debug)
-
-        def create_config_tree(self, src_dir, dest_dir, script):
-            """
-            Append commands to script that will create the directory hierarchy
-            on the remote federant.
-            """
-
-            if os.path.isdir(src_dir):
-                print >>script, "mkdir -p %s" % dest_dir
-                print >>script, "chmod 770 %s" % dest_dir
-
-                for f in os.listdir(src_dir):
-                    if os.path.isdir(f):
-                        self.create_config_tree("%s/%s" % (src_dir, f), 
-                                "%s/%s" % (dest_dir, f), script)
-            else:
-                self.log.debug("[create_config_tree]: Not a directory: %s" \
-                        % src_dir)
-
-        def ship_configs(self, host, user, src_dir, dest_dir):
-            """
-            Copy federant-specific configuration files to the federant.
-            """
-            for f in os.listdir(src_dir):
-                if os.path.isdir(f):
-                    if not self.ship_configs(host, user, "%s/%s" % (src_dir, f), 
-                            "%s/%s" % (dest_dir, f)):
-                        return False
-                else:
-                    if not self.scp_file("%s/%s" % (src_dir, f), 
-                            user, host, dest_dir):
-                        return False
-            return True
-
-        def get_state(self, user, host, tb, pid, eid):
+            access.proxy_emulab_segment.__init__(self, log=log, 
+		    keyfile=keyfile, debug=debug)
+	    self.null = """
+set ns [new Simulator]
+source tb_compat.tcl
+
+set a [$ns node]
+
+$ns rtproto Session
+$ns run 
+"""
+
+        def get_state(self, user, host, pid, eid):
             # command to test experiment state
             expinfo_exec = "/usr/testbed/bin/expinfo"  
@@ -939,6 +922,7 @@
                 rv = 0
             else:
-                status = Popen(cmd, stdout=PIPE, stderr=dev_null,
-                        close_fds=True)
+		self.log.debug("Checking state")
+                status = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+			stderr=dev_null, close_fds=True)
                 for line in status.stdout:
                     m = state_re.match(line)
@@ -956,14 +940,14 @@
             if rv != 0 and state != 'none':
                 raise service_error(service_error.internal,
-                        "Cannot get status of segment %s:%s/%s" % \
-                                (tb, pid, eid))
+                        "Cannot get status of segment:%s/%s" % (pid, eid))
             elif state not in ('active', 'swapped', 'swapping', 'none'):
                 raise service_error(service_error.internal,
-                        "Cannot get status of segment %s:%s/%s" % \
-                                (tb, pid, eid))
-            else: return state
-
-
-        def __call__(self, tb, eid, tbparams, tmpdir, timeout=0):
+                        "Cannot get status of segment:%s/%s" % (pid, eid))
+            else:
+		self.log.debug("State is %s" % state)
+		return state
+
+
+        def __call__(self, parent, eid, pid, user, tclfile, tmpdir, timeout=0):
             """
             Start a sub-experiment on a federant.
@@ -974,22 +958,14 @@
             """
             # ops node in the federant
-            host = "%s%s" % (tbparams[tb]['host'], tbparams[tb]['domain'])
-            user = tbparams[tb]['user']     # federant user
-            pid = tbparams[tb]['project']   # federant project
-            # XXX
-            base_confs = ( "hosts",)
-            tclfile = "%s.%s.tcl" % (eid, tb)   # sub-experiment description
+            host = "%s%s" % (parent.ops, parent.domain)
             # Configuration directories on the remote machine
             proj_dir = "/proj/%s/exp/%s/tmp" % (pid, eid)
-            tarfiles_dir = "/proj/%s/tarfiles/%s" % (pid, eid)
-            rpms_dir = "/proj/%s/rpms/%s" % (pid, eid)
-
-            state = self.get_state(user, host, tb, pid, eid)
-
-            self.log.debug("[start_segment]: %s: %s" % (tb, state))
-            self.log.info("[start_segment]:transferring experiment to %s" % tb)
-
-            if not self.scp_file("%s/%s/%s" % \
-                    (tmpdir, tb, tclfile), user, host):
+            softdir = "/proj/%s/software/%s" % (pid, eid)
+	    # Local software dir
+	    lsoftdir = "%s/software" % tmpdir
+
+            state = self.get_state(user, host, pid, eid)
+
+            if not self.scp_file(tclfile, user, host):
                 return False
             
@@ -998,7 +974,15 @@
                 # logs there if the modify fails.  Emulab software discards the
                 # logs from a failed startexp
+		try:
+		    f = open("%s/null.tcl" % tmpdir, "w")
+		    print >>f, self.null
+		    f.close()
+		except IOError, e:
+		    raise service_error(service_error.internal,
+			    "Cannot stage tarfile/rpm: %s" % e.strerror)
+
                 if not self.scp_file("%s/null.tcl" % tmpdir, user, host):
                     return False
-                self.log.info("[start_segment]: Creating %s on %s" % (eid, tb))
+                self.log.info("[start_segment]: Creating %s" % eid)
                 timedout = False
                 try:
@@ -1012,8 +996,7 @@
 
                 if timedout:
-                    state = self.get_state(user, host, tb, pid, eid)
+                    state = self.get_state(user, host, pid, eid)
                     if state != "swapped":
                         return False
-
             
             # Open up a temporary file to contain a script for setting up the
@@ -1030,17 +1013,9 @@
             # Script the filesystem changes
             print >>scriptfile, "/bin/rm -rf %s" % proj_dir
-            # Clear and create the tarfiles and rpm directories
-            for d in (tarfiles_dir, rpms_dir):
-                print >>scriptfile, "/bin/rm -rf %s/*" % d
-                print >>scriptfile, "mkdir -p %s" % d
+            # Clear and create the software directory
+	    print >>scriptfile, "/bin/rm -rf %s/*" % softdir
             print >>scriptfile, 'mkdir -p %s' % proj_dir
-            self.create_config_tree("%s/%s" % (tmpdir, tb),
-                    proj_dir, scriptfile)
-            if os.path.isdir("%s/tarfiles" % tmpdir):
-                self.create_config_tree("%s/tarfiles" % tmpdir, tarfiles_dir,
-                        scriptfile)
-            if os.path.isdir("%s/rpms" % tmpdir):
-                self.create_config_tree("%s/rpms" % tmpdir, rpms_dir, 
-                        scriptfile)
+            if os.path.isdir(lsoftdir):
+		print >>scriptfile, 'mkdir -p %s' % softdir
             print >>scriptfile, "rm -f %s" % scriptbase
             scriptfile.close()
@@ -1057,26 +1032,22 @@
                 return False
 
-            for f in base_confs:
-                if not self.scp_file("%s/%s" % (tmpdir, f), user, host,
-                        "%s/%s" % (proj_dir, f)):
-                    return False
-            if not self.ship_configs(host, user, "%s/%s" % (tmpdir, tb),
-                    proj_dir):
-                return False
-            if os.path.isdir("%s/tarfiles" % tmpdir):
-                if not self.ship_configs(host, user,
-                        "%s/tarfiles" % tmpdir, tarfiles_dir):
-                    return False
-            if os.path.isdir("%s/rpms" % tmpdir):
-                if not self.ship_configs(host, user,
-                        "%s/rpms" % tmpdir, tarfiles_dir):
-                    return False
+            for f in os.listdir(tmpdir):
+		if not os.path.isdir("%s/%s" % (tmpdir, f)):
+		    if not self.scp_file("%s/%s" % (tmpdir, f), user, host,
+			    "%s/%s" % (proj_dir, f)):
+			return False
+            if os.path.isdir(lsoftdir):
+		for f in os.listdir(lsoftdir):
+		    if not os.path.isdir("%s/%s" % (lsoftdir, f)):
+			if not self.scp_file("%s/%s" % (lsoftdir, f), 
+				user, host, "%s/%s" % (softdir, f)):
+			    return False
             # Stage the new configuration (active experiments will stay swapped
             # in now)
-            self.log.info("[start_segment]: Modifying %s on %s" % (eid, tb))
+            self.log.info("[start_segment]: Modifying %s" % eid)
             try:
                 if not self.ssh_cmd(user, host,
                         "/usr/testbed/bin/modexp -r -s -w %s %s %s" % \
-                                (pid, eid, tclfile),
+                                (pid, eid, tclfile.rpartition('/')[2]),
                         "modexp", timeout= 60 * 10):
                     return False
@@ -1088,6 +1059,5 @@
             # Active experiments are still swapped, this swaps the others in.
             if state != 'active':
-                self.log.info("[start_segment]: Swapping %s in on %s" % \
-                        (eid, tb))
+                self.log.info("[start_segment]: Swapping %s" % eid)
                 timedout = False
                 try:
@@ -1104,5 +1074,5 @@
                     self.log.debug("[start_segment]: swapin timed out " +\
                             "checking state")
-                    state = self.get_state(user, host, tb, pid, eid)
+                    state = self.get_state(user, host, pid, eid)
                     self.log.debug("[start_segment]: state is %s" % state)
                     return state == 'active'
@@ -1110,5 +1080,5 @@
             return True
 
-    class stop_segment(emulab_segment):
+    class stop_segment(proxy_emulab_segment):
         def __init__(self, log=None, keyfile=None, debug=False):
             experiment_control_local.emulab_segment.__init__(self,
@@ -1137,5 +1107,7 @@
 
     def generate_portal_configs(self, topo, pubkey_base, secretkey_base, 
-	    tmpdir):
+	    tmpdir, master):
+
+	seer_out = False
 	for p in [ e for e in topo.elements \
 		if isinstance(e, topdl.Computer) and e.get_attribute('portal')]:
@@ -1145,7 +1117,10 @@
 	    lproj, leid = lexp.split('/', 1)
 	    ldomain = e.get_attribute('domain')
+	    mexp = e.get_attribute('masterexperiment')
+	    mproj, meid = mexp.split("/", 1)
+	    mdomain = e.get_attribute('masterdomain')
 	    scriptdir = e.get_attribute('scriptdir')
 	    active = e.get_attribute('active')
-	    type = e.get_attribute('type')
+	    type = e.get_attribute('portal_type')
 	    segid = fedid(hexstr=e.get_attribute('peer_segment'))
 	    for e in topo.elements:
@@ -1170,5 +1145,5 @@
 		print >>f, "RemoteEventServerName: event-server%s" % rdomain
 		print >>f, "SeerControl: control.%s.%s%s" % \
-			(leid.lower(), lproj.lower(), ldomain)
+			(meid.lower(), mproj.lower(), mdomain)
 		print >>f, "Type: %s" % type
 		print >>f, "RemoteExperiment: %s" % rexp
@@ -1188,4 +1163,83 @@
 		raise service_error(service_error.internal,
 			"Can't write protal config %s: %s" % (cfn, e))
+	    
+	    # XXX: This little seer config file needs to go away.
+	    if not seer_out:
+		try:
+		    seerfn = "%s/seer.conf" % tmpdir
+		    f = open(seerfn, "w")
+		    if not master:
+			print >>f, "ControlNode: control.%s.%s%s" % \
+			    (meid.lower(), mproj.lower(), mdomain)
+		    print >>f, "ExperimentID: %s" % mexp
+		    f.close()
+		except IOError, e:
+		    raise service_error(service_error.internal, 
+			    "Can't write seer.conf: %s" %e)
+		seer_out = True
+
+
+    def generate_client_conf(self, user, proj, eid, tmpdir):
+	try:
+	    f = open("%s/client.conf" % tmpdir, "w")
+	    if self.attrs.has_key('SMBshare'):
+		print >>f, "SMBshare: %s" % self.attrs['SMBshare']
+	    print >>f, "ProjectUser: %s" % user
+	    print >>f, "ProjectName: %s" % proj
+	    print >>f, "ExperimentID: %s/%s" % (proj, eid)
+	    f.close()
+	except IOError, e:
+	    raise service_error(service_error.internal,
+		    "Cannot write client.conf: %s" %s)
+
+    def generate_ns2(self, topo, expfn, softdir, master):
+	t = topo.clone()
+
+	# Weed out the things we aren't going to instantiate: Segments, portal
+	# substrates, and portal interfaces.  (The copi in the for loop allows
+	# us to delete from e.elements in side the for loop).
+	for e in [e for e in t.elements]:
+	    if isinstance(e, topdl.Segment):
+		t.elements.remove(e)
+	    if isinstance(e, topdl.Computer):
+		e.interface = [i for i in e.interface \
+			if not i.get_attribute('portal')]
+	t.substrates = [ s for s in t.substrates \
+		if not s.get_attribute('portal')]
+	t.incorporate_elements()
+
+	# Localize the software locations
+	for e in t.elements:
+	    for s in getattr(e, 'software', []):
+		s.location = re.sub("^.*/", softdir, s.location)
+
+
+	# Customize the ns2 output for local portal commands and images
+	filters = []
+
+	if master: cmdname = 'MasterConnectorCmd'
+	else:cmdname = 'SlaveConnectorCmd'
+
+	if self.attrs.has_key(cmdname):
+	    filters.append(topdl.generate_portal_command_filter(
+		self.attrs.get(cmdname)))
+
+	if self.attrs.has_key('connectorImage'):
+	    filters.append(topdl.generate_portal_image_filter(
+		self.attrs.get('connectorImage')))
+
+	if self.attrs.has_key('connectorType'):
+	    filters.append(topdl.generate_portal_hardware_filter(
+		self.attrs.get('connectorType')))
+
+	# Convert to ns and write it out
+	expfile = topdl.topology_to_ns2(t, filters)
+	try:
+	    f = open(expfn, "w")
+	    print >>f, expfile
+	    f.close()
+	except IOError:
+	    raise service_error(service_error.internal,
+		    "Cannot write experiment file %s: %s" % (expfn,e))
 
     def StartSegment(self, req, fid):
@@ -1214,6 +1268,4 @@
 
 	configs = set(('hosts', 'ssh_pubkey', 'ssh_secretkey'))
-	keys = set(('ssh_pubkey', 'ssh_secretkey'))
-
 
 	try:
@@ -1221,12 +1273,10 @@
 	except KeyError:
 	    raise service_error(server_error.req, "Badly formed request")
+
 	auth_attr = req['allocID']['fedid']
+	aid = "%s" % auth_attr
 	attrs = req.get('fedAttr', [])
-	print auth_attr
-	print "%s" % auth_attr
-	if self.auth.check_attribute(fid, auth_attr):
-	    print "OK"
-	else:
-	    print "Fail"
+	if not self.auth.check_attribute(fid, auth_attr):
+	    raise service_error(service_error.access, "Access denied")
 
 	if req.has_key('segmentdescription') and \
@@ -1238,16 +1288,21 @@
 		    "Request missing segmentdescription'")
 
+	master = req.get('master', False)
+
 	certfile = "%s/%s.pem" % (self.certdir, auth_attr)
         try:
             tmpdir = tempfile.mkdtemp(prefix="access-")
+	    softdir = "%s/software" % tmpdir
         except IOError:
             raise service_error(service_error.internal, "Cannot create tmp dir")
 
 	sw = set()
-	for e in [c for c in topo.elements if getattr(c, 'software', False)]:
-	    for s in e.software:
+	for e in topo.elements:
+	    for s in getattr(e, 'software', []):
 		sw.add(s.location)
+	if len(sw) > 0:
+	    os.mkdir(softdir)
 	for s in sw:
-	    get_url(s, certfile, tmpdir)
+	    get_url(s, certfile, softdir)
 
 	for a in attrs:
@@ -1255,9 +1310,37 @@
 		get_url(a['value'], certfile, tmpdir)
 	    if a['attribute'] == 'ssh_pubkey':
-		pubkey_base = a['attribute'].rpartition('/')[2]
+		pubkey_base = a['value'].rpartition('/')[2]
 	    if a['attribute'] == 'ssh_secretkey':
-		secretkey_base = a['attribute'].rpartition('/')[2]
-
-	self.generate_portal_configs(topo, pubkey_base, secretkey_base, tmpdir)
+		secretkey_base = a['value'].rpartition('/')[2]
+	    if a['attribute'] == 'experiment_name':
+		ename = a['value']
+
+	proj = None
+	user = None
+	self.state_lock.acquire()
+	if self.allocation.has_key(aid):
+	    proj = self.allocation[aid].get('project', None)
+	    if not proj: 
+		proj = self.allocation[aid].get('sproject', None)
+	    user = self.allocation[aid].get('user', None)
+	self.state_lock.release()
+
+	if not proj:
+	    raise service_error(service_error.internal, 
+		    "Can't find project for %s" %aid)
+
+	if not user:
+	    raise service_error(service_error.internal, 
+		    "Can't find creation user for %s" %aid)
+
+	expfile = "%s/experiment.tcl" % tmpdir
+
+	self.generate_portal_configs(topo, pubkey_base, 
+		secretkey_base, tmpdir, master)
+	self.generate_ns2(topo, expfile, 
+		"/proj/%s/software/%s/" % (proj, ename), master)
+	self.generate_client_conf(user, proj, ename, tmpdir)
+	starter = self.start_segment(keyfile=self.ssh_privkey_file, debug=False)
+	starter(self, ename, proj, user, expfile, tmpdir)
 
 	return { 'allocID': req['allocID'] }
