Index: fedd/Makefile/federation/__init__.py
===================================================================
--- fedd/Makefile/federation/__init__.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/__init__.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,7 @@
+#/usr/local/bin/python
+
+from fedid import fedid
+from service_error import service_error
+from config_parser import config_parser
+
+import parse_detail
Index: fedd/Makefile/federation/abac_remote_service.py
===================================================================
--- fedd/Makefile/federation/abac_remote_service.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/abac_remote_service.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,58 @@
+#!/usr/local/bin/python
+
+import fedd_abac_services
+#import fedd_services
+import remote_service
+from fedd_services import CreateRequestMessage
+from M2Crypto import SSL
+from remote_service import service_caller
+from service_error import service_error
+from server import soap_handler
+
+# Turn off the matching of hostname to certificate ID
+SSL.Connection.clientPostConnectionCheck = None
+
+class abac_soap_handler(soap_handler):
+    """
+    Encapsulate the handler code to unpack and pack SOAP requests and     
+    responses and call the given method.
+
+    The code to decapsulate and encapsulate parameters encoded in SOAP is the
+    same modulo a few parameters.  This is a functor that calls a fedd service
+    trhough a soap interface.  The parameters are the typecode of the request
+    parameters, the method to call (usually a bound instance of a method on a
+    fedd service providing class), the constructor of a response packet and
+    name of the body element of that packet.  The handler takes a ParsedSoap
+    object (the request) and returns an instance of the class created by
+    constructor containing the response.  Failures of the constructor or badly
+    created constructors will result in None being returned.
+    """
+
+    def get_class(self, class_name):
+	return getattr(fedd_abac_services, class_name, None) or \
+		getattr(fedd_abac_internal_services, class_name, None)
+
+
+class abac_service_caller(service_caller):
+    def __init__(self, service_name, request_message=None, 
+	    request_body_name=None, tracefile=None):
+	self.service_name = service_name
+        self.locator = fedd_abac_services.feddABACServiceLocator
+	self.port_name = 'getfeddABACPortType'
+
+	if request_message: self.request_message = request_message
+	else:
+	    request_message_name = "%sRequestMessage" % service_name
+	    self.request_message = \
+		    getattr(fedd_abac_services, request_message_name, None) # or \
+#		    getattr(fedd_internal_services, request_message_name, None)
+	    if not self.request_message:
+		raise service_error(service_error.internal,
+			"Cannot find class for %s" % request_message_name)
+
+	if request_body_name: self.request_body_name = request_body_name
+	else: self.request_body_name = "%sRequestBody" % service_name
+
+	self.tracefile = tracefile
+	self.__call__ = self.call_service
+
Index: fedd/Makefile/federation/access.py
===================================================================
--- fedd/Makefile/federation/access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,778 @@
+#!/usr/local/bin/python
+
+import os,sys
+import stat # for chmod constants
+import re
+import random
+import string
+import copy
+import pickle
+import logging
+import subprocess
+
+from threading import *
+from M2Crypto.SSL import SSLError
+
+from util import *
+from allocate_project import allocate_project_local, allocate_project_remote
+from access_project import access_project
+from fedid import fedid, generate_fedid
+from authorizer import authorizer
+from service_error import service_error
+from remote_service import xmlrpc_handler, soap_handler, service_caller
+
+import httplib
+import tempfile
+from urlparse import urlparse
+
+import topdl
+import list_log
+import proxy_emulab_segment
+import local_emulab_segment
+
+
+# Make log messages disappear if noone configures a fedd logger
+class nullHandler(logging.Handler):
+    def emit(self, record): pass
+
+fl = logging.getLogger("fedd.access")
+fl.addHandler(nullHandler())
+
+class access_base:
+    """
+    The implementation of access control based on mapping users to projects.
+
+    Users can be mapped to existing projects or have projects created
+    dynamically.  This implements both direct requests and proxies.
+    """
+
+    class parse_error(RuntimeError): pass
+
+    def __init__(self, config=None, auth=None):
+        """
+        Initializer.  Pulls parameters out of the ConfigParser's access section.
+        """
+
+        # Make sure that the configuration is in place
+        if not config: 
+            raise RunTimeError("No config to fedd.access")
+
+        self.project_priority = config.getboolean("access", "project_priority")
+
+	self.certdir = config.get("access","certdir")
+	self.create_debug = config.getboolean("access", "create_debug")
+	self.cleanup = not config.getboolean("access", "leave_tmpfiles")
+	self.access_type = config.get("access", "type")
+        self.log = logging.getLogger("fedd.access")
+        set_log_level(config, "access", self.log)
+        self.state_lock = Lock()
+	self.state = { }
+	# subclasses fill with what and how they export.
+	self.exports = { }
+	# XXX: Configurable
+	self.imports = set(('SMB', 'seer', 'userconfig', 'seer_master',
+	    'hide_hosts'))
+
+        if auth: self.auth = auth
+        else:
+            self.log.error(\
+                    "[access]: No authorizer initialized, creating local one.")
+            auth = authorizer()
+
+        self.state_filename = config.get("access", "access_state")
+        self.read_state()
+
+        # Keep cert_file and cert_pwd coming from the same place
+        self.cert_file = config.get("access", "cert_file")
+        if self.cert_file:
+            self.cert_pwd = config.get("access", "cert_pw")
+        else:
+            self.cert_file = config.get("globals", "cert_file")
+            self.sert_pwd = config.get("globals", "cert_pw")
+
+        self.trusted_certs = config.get("access", "trusted_certs") or \
+                config.get("globals", "trusted_certs")
+
+
+    @staticmethod
+    def software_list(v):
+	"""
+	From a string containing a sequence of space separated pairs, return a
+	list of tuples with pairs of location and file.
+	"""
+	l = [ ]
+	if v:
+	    ps = v.split(" ")
+	    while len(ps):
+		loc, file = ps[0:2]
+		del ps[0:2]
+		l.append((loc, file))
+	return l
+
+    @staticmethod
+    def add_kit(e, kit):
+	"""
+	Add a Software object created from the list of (install, location)
+	tuples passed as kit  to the software attribute of an object e.  We
+	do this enough to break out the code, but it's kind of a hack to
+	avoid changing the old tuple rep.
+	"""
+
+	s = [ topdl.Software(install=i, location=l) for i, l in kit]
+
+	if isinstance(e.software, list): e.software.extend(s)
+	else: e.software = s 
+
+
+    def read_access(self, config, access_obj=None):
+        """
+	Read an access DB with filename config  of the form:
+	    (id, id, id) -> attribute, something
+	where the ids can be fedids, strings, or <any> or <none>, attribute is
+	the attribute to assign , and something is any set of charcters.  The
+	hash self.access is populated with mappings from those triples to the
+	results of access_obj being called on the remainder of the line (if
+	present).  If access_obj is not given, the string itself is entered in
+	the hash.  Additionally, a triple with <any> and <none> mapped to None
+	is entered in self.auth with the attribute given.
+
+	Parsing errors result in a self.parse_error exception being raised.
+	access_obj should throw that as well.
+        """
+        lineno=0
+        name_expr = "["+string.ascii_letters + string.digits + "\.\-_]+"
+        fedid_expr = "fedid:[" + string.hexdigits + "]+"
+        key_name = "(<ANY>|<NONE>|"+fedid_expr + "|"+ name_expr + ")"
+        access_re = re.compile('\('+key_name+'\s*,\s*'+key_name+'\s*,\s*'+
+                key_name+'\s*\)\s*->\s*([^,]+)\s*(.*)', re.IGNORECASE)
+
+        def parse_name(n):
+            if n.startswith('fedid:'): return fedid(hexstr=n[len('fedid:'):])
+            else: return n
+        
+        def auth_name(n):
+            if isinstance(n, basestring):
+                if n =='<any>' or n =='<none>': return None
+                else: return unicode(n)
+            else:
+                return n
+	def strip_comma(s):
+	    s = s.strip()
+	    if s.startswith(','):
+		s = s[1:].strip()
+	    return s
+
+	if access_obj is None:
+	    access_obj = lambda(x): "%s" % x
+
+        f = open(config, "r");
+	try:
+	    for line in f:
+		lineno += 1
+		line = line.strip();
+		if len(line) == 0 or line.startswith('#'):
+		    continue
+
+		# Access line (t, p, u) -> anything
+		m = access_re.match(line)
+		if m != None:
+		    access_key = tuple([ parse_name(x) \
+			    for x in m.group(1,2,3)])
+		    attribute = m.group(4)
+		    auth_key = tuple([ auth_name(x) for x in access_key])
+		    self.auth.set_attribute(auth_key, attribute)
+		    if len(m.group(5)) > 0:
+			access_val = access_obj(strip_comma(m.group(5)))
+			self.access[access_key] = access_val
+		    continue
+
+		# Nothing matched to here: unknown line - raise exception
+		f.close()
+		raise self.parse_error(
+			"Unknown statement at line %d of %s" % \
+			(lineno, config))
+	finally:
+	    if f: f.close()
+
+    def get_users(self, obj):
+        """
+        Return a list of the IDs of the users in dict
+        """
+        if obj.has_key('user'):
+            return [ unpack_id(u['userID']) \
+                    for u in obj['user'] if u.has_key('userID') ]
+        else:
+            return None
+
+    def write_state(self):
+        if self.state_filename:
+            try:
+                f = open(self.state_filename, 'w')
+                pickle.dump(self.state, f)
+		self.log.debug("Wrote state to %s" % self.state_filename)
+            except EnvironmentError, e:
+                self.log.error("Can't write file %s: %s" % \
+                        (self.state_filename, e))
+            except pickle.PicklingError, e:
+                self.log.error("Pickling problem: %s" % e)
+            except TypeError, e:
+                self.log.error("Pickling problem (TypeError): %s" % e)
+
+
+    def read_state(self):
+        """
+        Read a new copy of access state.  Old state is overwritten.
+
+        State format is a simple pickling of the state dictionary.
+        """
+        if self.state_filename:
+            try:
+                f = open(self.state_filename, "r")
+                self.state = pickle.load(f)
+		self.log.debug("[read_state]: Read state from %s" % \
+			self.state_filename)
+            except EnvironmentError, e:
+                self.log.warning(("[read_state]: No saved state: " +\
+                        "Can't open %s: %s") % (self.state_filename, e))
+            except EOFError, e:
+                self.log.warning(("[read_state]: " +\
+                        "Empty or damaged state file: %s:") % \
+                        self.state_filename)
+            except pickle.UnpicklingError, e:
+                self.log.warning(("[read_state]: No saved state: " + \
+                        "Unpickling failed: %s") % e)
+
+
+
+    def permute_wildcards(self, a, p):
+        """Return a copy of a with various fields wildcarded.
+
+        The bits of p control the wildcards.  A set bit is a wildcard
+        replacement with the lowest bit being user then project then testbed.
+        """
+        if p & 1: user = ["<any>"]
+        else: user = a[2]
+        if p & 2: proj = "<any>"
+        else: proj = a[1]
+        if p & 4: tb = "<any>"
+        else: tb = a[0]
+
+        return (tb, proj, user)
+
+    def find_access(self, search):
+        """
+        Search the access DB for a match on this tuple.  Return the matching
+        access tuple and the user that matched.
+        
+        NB, if the initial tuple fails to match we start inserting wildcards in
+        an order determined by self.project_priority.  Try the list of users in
+        order (when wildcarded, there's only one user in the list).
+        """
+        if self.project_priority: perm = (0, 1, 2, 3, 4, 5, 6, 7)
+        else: perm = (0, 2, 1, 3, 4, 6, 5, 7)
+
+        for p in perm: 
+            s = self.permute_wildcards(search, p)
+            # s[2] is None on an anonymous, unwildcarded request
+            if s[2] != None:
+                for u in s[2]:
+                    if self.access.has_key((s[0], s[1], u)):
+                        return (self.access[(s[0], s[1], u)], u)
+            else:
+                if self.access.has_key(s):
+                    return (self.access[s], None)
+        return None, None
+
+    def lookup_access_base(self, req, fid):
+        """
+        Determine the allowed access for this request.  Return the access and
+        which fields are dynamic.
+
+        The fedid is needed to construct the request
+        """
+	user_re = re.compile("user:\s(.*)")
+	project_re = re.compile("project:\s(.*)")
+
+        # Search keys
+	tb = fid
+	user = [ user_re.findall(x)[0] for x in req.get('credential', []) \
+		if user_re.match(x)]
+	project = [ project_re.findall(x)[0] \
+		for x in req.get('credential', []) \
+		    if project_re.match(x)]
+
+	if len(project) == 1: project = project[0]
+	elif len(project) == 0: project = None
+	else: 
+	    raise service_error(service_error.req, 
+		    "More than one project credential")
+
+        # Confirm authorization
+        for u in user:
+            self.log.debug("[lookup_access] Checking access for %s" % \
+                    ((tb, project, u),))
+            if self.auth.check_attribute((tb, project, u), 'access'):
+                self.log.debug("[lookup_access] Access granted")
+                break
+            else:
+                self.log.debug("[lookup_access] Access Denied")
+        else:
+            raise service_error(service_error.access, "Access denied")
+
+        # This maps a valid user to the Emulab projects and users to use
+        found, user_match = self.find_access((tb, project, user))
+
+	return (found, (tb, project, user_match))
+        
+
+    def get_handler(self, path, fid):
+	self.log.info("Get handler %s %s" % (path, fid))
+	if self.auth.check_attribute(fid, path) and self.userconfdir:
+	    return ("%s/%s" % (self.userconfdir, path), "application/binary")
+	else:
+	    return (None, None)
+
+    def export_userconf(self, project):
+	dev_null = None
+	confid, confcert = generate_fedid("test", dir=self.userconfdir, 
+		log=self.log)
+	conffilename = "%s/%s" % (self.userconfdir, str(confid))
+	cf = None
+	try:
+	    cf = open(conffilename, "w")
+	    os.chmod(conffilename, stat.S_IRUSR | stat.S_IWUSR)
+	except EnvironmentError, e:
+	    raise service_error(service_error.internal, 
+		    "Cannot create user configuration data")
+
+	try:
+	    dev_null = open("/dev/null", "a")
+	except EnvironmentError, e:
+	    self.log.error("export_userconf: can't open /dev/null: %s" % e)
+
+	cmd = "%s %s" % (self.userconfcmd, project)
+	conf = subprocess.call(cmd.split(" "),
+		stdout=cf, stderr=dev_null, close_fds=True)
+
+	self.auth.set_attribute(confid, "/%s" % str(confid))
+
+	return confid, confcert
+
+    def export_SMB(self, id, state, project, user, attrs):
+	if project and user:
+	    return [{ 
+		    'id': id,
+		    'name': 'SMB',
+		    'visibility': 'export',
+		    'server': 'http://fs:139',
+		    'fedAttr': [
+			    { 'attribute': 'SMBSHARE', 'value': 'USERS' },
+			    { 'attribute': 'SMBUSER', 'value': user },
+			    { 'attribute': 'SMBPROJ', 'value': project },
+			]
+		    }]
+	else:
+	    self.log.warn("Cannot export SMB w/o user and project")
+	    return [ ]
+
+    def export_seer(self, id, state, project, user, attrs):
+	return [{ 
+		'id': id,
+		'name': 'seer',
+		'visibility': 'export',
+		'server': 'http://control:16606',
+		}]
+
+    def export_local_seer(self, id, state, project, user, attrs):
+	return [{ 
+		'id': id,
+		'name': 'local_seer_control',
+		'visibility': 'export',
+		'server': 'http://control:16606',
+		}]
+
+    def export_seer_master(self, id, state, project, user, attrs):
+	return [{ 
+		'id': id,
+		'name': 'seer_master',
+		'visibility': 'export',
+		'server': 'http://seer-master:17707',
+		}]
+
+    def export_tmcd(self, id, state, project, user, attrs):
+	return [{ 
+		'id': id,
+		'name': 'seer',
+		'visibility': 'export',
+		'server': 'http://boss:7777',
+		}]
+
+    def export_userconfig(self, id, state, project, user, attrs):
+	if self.userconfdir and self.userconfcmd \
+		and self.userconfurl:
+	    cid, cert = self.export_userconf(project)
+	    state['userconfig'] = unicode(cid)
+	    return [{
+		    'id': id,
+		    'name': 'userconfig',
+		    'visibility': 'export',
+		    'server': "%s/%s" % (self.userconfurl, str(cid)),
+		    'fedAttr': [
+			{ 'attribute': 'cert', 'value': cert },
+		    ]
+		    }]
+	else:
+	    return [ ]
+
+    def export_hide_hosts(self, id, state, project, user, attrs):
+	return [{
+		'id': id, 
+		'name': 'hide_hosts',
+		'visibility': 'export',
+		'fedAttr': [ x for x in attrs \
+			if x.get('attribute', "") == 'hosts'],
+		}]
+
+    def export_project_export(self, id, state, project, user, attrs):
+	rv = [ ]
+	rv.extend(self.export_SMB(id, state, project, user, attrs))
+	rv.extend(self.export_userconfig(id, state, project, user, attrs))
+	return rv
+
+    def export_services(self, sreq, project=None, user=None):
+	exp = [ ]
+	state = { }
+	for s in sreq:
+	    sname = s.get('name', '')
+	    svis = s.get('visibility', '')
+	    sattrs = s.get('fedAttr', [])
+	    if svis == 'export':
+		if sname in self.exports:
+		    id = s.get('id', 'no_id')
+		    exp.extend(self.exports[sname](id, state, project, user,
+			    sattrs))
+
+	return (exp, state)
+
+    def build_access_response(self, alloc_id, ap, services):
+        """
+        Create the SOAP response.
+
+        Build the dictionary description of the response and use
+        fedd_utils.pack_soap to create the soap message.  ap is the allocate
+        project message returned from a remote project allocation (even if that
+        allocation was done locally).
+        """
+        # Because alloc_id is already a fedd_services_types.IDType_Holder,
+        # there's no need to repack it
+        msg = { 
+                'allocID': alloc_id,
+		'fedAttr': [
+		    { 'attribute': 'domain', 'value': self.domain } , 
+		    { 'attribute': 'project', 'value': 
+			ap['project'].get('name', {}).get('localname', "???") },
+		]
+            }
+
+	if self.dragon_endpoint:
+	    msg['fedAttr'].append({'attribute': 'dragon',
+		'value': self.dragon_endpoint})
+	if self.deter_internal:
+	    msg['fedAttr'].append({'attribute': 'deter_internal',
+		'value': self.deter_internal})
+	#XXX: ??
+	if self.dragon_vlans:
+	    msg['fedAttr'].append({'attribute': 'vlans',
+		'value': self.dragon_vlans})
+
+	if services:
+	    msg['service'] = services
+        return msg
+
+    def generate_portal_configs(self, topo, pubkey_base, secretkey_base, 
+	    tmpdir, lproj, leid, connInfo, services):
+
+	def conninfo_to_dict(key, info):
+	    """
+	    Make a cpoy of the connection information about key, and flatten it
+	    into a single dict by parsing out any feddAttrs.
+	    """
+
+	    rv = None
+	    for i in info:
+		if key == i.get('portal', "") or \
+			key in [e.get('element', "") \
+			for e in i.get('member', [])]:
+		    rv = i.copy()
+		    break
+
+	    else:
+		return rv
+
+	    if 'fedAttr' in rv:
+		for a in rv['fedAttr']:
+		    attr = a.get('attribute', "")
+		    val = a.get('value', "")
+		    if attr and attr not in rv:
+			rv[attr] = val
+		del rv['fedAttr']
+	    return rv
+
+	# XXX: un hardcode this
+	def client_null(f, s):
+	    print >>f, "Service: %s" % s['name']
+
+	def client_seer_master(f, s):
+	    print >>f, 'PortalAlias: seer-master'
+
+	def client_smb(f, s):
+	    print >>f, "Service: %s" % s['name']
+	    smbshare = None
+	    smbuser = None
+	    smbproj = None
+	    for a in s.get('fedAttr', []):
+		if a.get('attribute', '') == 'SMBSHARE':
+		    smbshare = a.get('value', None)
+		elif a.get('attribute', '') == 'SMBUSER':
+		    smbuser = a.get('value', None)
+		elif a.get('attribute', '') == 'SMBPROJ':
+		    smbproj = a.get('value', None)
+
+	    if all((smbshare, smbuser, smbproj)):
+		print >>f, "SMBshare: %s" % smbshare
+		print >>f, "ProjectUser: %s" % smbuser
+		print >>f, "ProjectName: %s" % smbproj
+
+	def client_hide_hosts(f, s):
+	    for a in s.get('fedAttr', [ ]):
+		if a.get('attribute', "") == 'hosts':
+		    print >>f, "Hide: %s" % a.get('value', "")
+
+	client_service_out = {
+		'SMB': client_smb,
+		'tmcd': client_null,
+		'seer': client_null,
+		'userconfig': client_null,
+		'project_export': client_null,
+		'seer_master': client_seer_master,
+		'hide_hosts': client_hide_hosts,
+	    }
+
+	def client_seer_master_export(f, s):
+	    print >>f, "AddedNode: seer-master"
+
+	def client_seer_local_export(f, s):
+	    print >>f, "AddedNode: control"
+
+	client_export_service_out = {
+		'seer_master': client_seer_master_export,
+		'local_seer_control': client_seer_local_export,
+	    }
+
+	def server_port(f, s):
+	    p = urlparse(s.get('server', 'http://localhost'))
+	    print >>f, 'port: remote:%s:%s:%s' % (p.port, p.hostname, p.port) 
+
+	def server_null(f,s): pass
+
+	def server_seer(f, s):
+	    print >>f, 'seer: True'
+
+	server_service_out = {
+		'SMB': server_port,
+		'tmcd': server_port,
+		'userconfig': server_null,
+		'project_export': server_null,
+		'seer': server_seer,
+		'seer_master': server_port,
+		'hide_hosts': server_null,
+	    }
+	# XXX: end un hardcode this
+
+
+	seer_out = False
+	client_out = False
+	mproj = None
+	mexp = None
+	control_gw = None
+	testbed = ""
+	# Create configuration files for the portals
+	for e in [ e for e in topo.elements \
+		if isinstance(e, topdl.Computer) and e.get_attribute('portal')]:
+	    myname = e.name
+	    type = e.get_attribute('portal_type')
+
+	    info = conninfo_to_dict(myname, connInfo)
+
+	    if not info:
+		raise service_error(service_error.req,
+			"No connectivity info for %s" % myname)
+
+	    peer = info.get('peer', "")
+	    ldomain = self.domain
+	    ssh_port = info.get('ssh_port', 22)
+
+	    # Collect this for the client.conf file
+	    if 'masterexperiment' in info:
+		mproj, meid = info['masterexperiment'].split("/", 1)
+
+	    if type in ('control', 'both'):
+		testbed = e.get_attribute('testbed')
+		control_gw = myname
+
+	    active = info.get('active', 'False')
+
+	    cfn = "%s/%s.gw.conf" % (tmpdir, myname.lower())
+	    tunnelconfig = self.tunnel_config
+	    try:
+		f = open(cfn, "w")
+		if active == 'True':
+		    print >>f, "active: True"
+		    print >>f, "ssh_port: %s" % ssh_port
+		    if type in ('control', 'both'):
+			for s in [s for s in services \
+				if s.get('name', "") in self.imports]:
+			    server_service_out[s['name']](f, s)
+
+		if tunnelconfig:
+		    print >>f, "tunnelip: %s" % tunnelconfig
+		print >>f, "peer: %s" % peer.lower()
+		print >>f, "ssh_pubkey: /proj/%s/exp/%s/tmp/%s" % \
+			(lproj, leid, pubkey_base)
+		print >>f, "ssh_privkey: /proj/%s/exp/%s/tmp/%s" % \
+			(lproj, leid, secretkey_base)
+		f.close()
+	    except EnvironmentError, e:
+		raise service_error(service_error.internal,
+			"Can't write protal config %s: %s" % (cfn, e))
+
+	# Done with portals, write the client config file.
+	try:
+	    f = open("%s/client.conf" % tmpdir, "w")
+	    if control_gw:
+		print >>f, "ControlGateway: %s.%s.%s%s" % \
+		    (myname.lower(), leid.lower(), lproj.lower(),
+			    ldomain.lower())
+	    for s in services:
+		if s.get('name',"") in self.imports and \
+			s.get('visibility','') == 'import':
+		    client_service_out[s['name']](f, s)
+		if s.get('name', '') in self.exports and \
+			s.get('visibility', '') == 'export' and \
+			s['name'] in client_export_service_out:
+		    client_export_service_out[s['name']](f, s)
+	    # Seer uses this.
+	    if mproj and meid:
+		print >>f, "ExperimentID: %s/%s" % (mproj, meid)
+	    f.close()
+	except EnvironmentError, e:
+	    raise service_error(service_error.internal,
+		    "Cannot write client.conf: %s" %s)
+
+    def configure_userconf(self, services, tmpdir):
+	"""
+	If the userconf service was imported, collect the configuration data.
+	"""
+	for s in services:
+	    s_name = s.get('name', '')
+	    s_vis = s.get('visibility','')
+	    if s_name  == 'userconfig' and s_vis == 'import':
+		# Collect ther server and certificate info.
+		u = s.get('server', None)
+		for a in s.get('fedAttr', []):
+		    if a.get('attribute',"") == 'cert':
+			cert = a.get('value', None)
+			break
+		else:
+		    cert = None
+
+		if cert:
+		    # Make a temporary certificate file for get_url.  The
+		    # finally clause removes it whether something goes
+		    # wrong (including an exception from get_url) or not.
+		    try:
+			tfos, tn = tempfile.mkstemp(suffix=".pem")
+			tf = os.fdopen(tfos, 'w')
+			print >>tf, cert
+			tf.close()
+			self.log.debug("Getting userconf info: %s" % u)
+			get_url(u, tn, tmpdir, "userconf")
+			self.log.debug("Got userconf info: %s" % u)
+		    except EnvironmentError, e:
+			raise service_error(service.error.internal, 
+				"Cannot create temp file for " + 
+				"userconfig certificates: %s" % e)
+		    except:
+			t, v, st = sys.exc_info()
+			raise service_error(service_error.internal,
+				"Error retrieving %s: %s" % (u, v))
+		    finally:
+			if tn: os.remove(tn)
+		else:
+		    raise service_error(service_error.req,
+			    "No certificate for retreiving userconfig")
+		break
+
+    def import_store_info(self, cf, connInfo):
+	"""
+	Pull any import parameters in connInfo in.  We translate them either
+	into known member names or fedAddrs.
+	"""
+
+	for c in connInfo:
+	    for p in [ p for p in c.get('parameter', []) \
+		    if p.get('type', '') == 'input']:
+		name = p.get('name', None)
+		key = p.get('key', None)
+		store = p.get('store', None)
+
+		if name and key and store :
+		    req = { 'name': key, 'wait': True }
+		    self.log.debug("Waiting for %s (%s) from %s" % \
+			    (name, key, store))
+		    r = self.call_GetValue(store, req, cf)
+		    r = r.get('GetValueResponseBody', None)
+		    if r :
+			if r.get('name', '') == key:
+			    v = r.get('value', None)
+			    if v is not None:
+				if name == 'peer':
+				    self.log.debug("Got peer %s" % v)
+				    c['peer'] = v
+				else:
+				    self.log.debug("Got %s %s" % (name, v))
+				    if c.has_key('fedAttr'):
+					c['fedAttr'].append({
+					    'attribute': name, 'value': v})
+				    else:
+					c['fedAttr']= [{
+					    'attribute': name, 'value': v}]
+			    else:
+				raise service_error(service_error.internal, 
+					'None value exported for %s'  % key)
+			else:
+			    raise service_error(service_error.internal, 
+				    'Different name returned for %s: %s' \
+					    % (key, r.get('name','')))
+		    else:
+			raise service_error(service_error.internal, 
+			    'Badly formatted response: no GetValueResponseBody')
+		else:
+		    raise service_error(service_error.internal, 
+			'Bad Services missing info for import %s' % c)
+
+    def remove_dirs(self, dir):
+	"""
+	Remove the directory tree and all files rooted at dir.  Log any errors,
+	but continue.
+	"""
+	self.log.debug("[removedirs]: removing %s" % dir)
+	try:
+	    for path, dirs, files in os.walk(dir, topdown=False):
+		for f in files:
+		    os.remove(os.path.join(path, f))
+		for d in dirs:
+		    os.rmdir(os.path.join(path, d))
+	    os.rmdir(dir)
+	except EnvironmentError, e:
+	    self.log.error("Error deleting directory tree in %s" % e);
Index: fedd/Makefile/federation/access_project.py
===================================================================
--- fedd/Makefile/federation/access_project.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/access_project.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,20 @@
+#!/usr/local/bin/python
+
+class access_project:
+    """
+    A project description used to grant access to this testbed.
+
+    The description includes a name and a list of node types to which the
+    project will be granted access.
+    """
+    def __init__(self, name, nt):
+	self.name = name
+	self.node_types = list(nt)
+
+    def __repr__(self):
+	if len(self.node_types) > 0:
+	    return "access_proj('%s', ['%s'])" % \
+		    (self.name, str("','").join(self.node_types))
+	else:
+	    return "access_proj('%s', [])" % self.name
+
Index: fedd/Makefile/federation/allocate_project.py
===================================================================
--- fedd/Makefile/federation/allocate_project.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/allocate_project.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,784 @@
+#!/usr/local/bin/python
+
+import os,sys
+import re
+import random
+import string
+import subprocess
+import threading
+import pickle
+import tempfile
+
+from util import *
+from fedid import fedid
+from remote_service import xmlrpc_handler, soap_handler, service_caller
+from service_error import service_error
+import logging
+
+
+# Configure loggers to dump to /dev/null which avoids errors if calling classes
+# don't configure them.
+class nullHandler(logging.Handler):
+    def emit(self, record): pass
+
+fl = logging.getLogger("fedd.allocate.local")
+fl.addHandler(nullHandler())
+fl = logging.getLogger("fedd.allocate.remote")
+fl.addHandler(nullHandler())
+
+
+class allocate_project_local:
+    """
+    Allocate projects on this machine in response to an access request.
+    """
+    dynamic_projects = 4
+    dynamic_keys= 2
+    confirm_keys = 1
+    none = 0
+
+    levels = {
+	    'dynamic_projects': dynamic_projects,
+	    'dynamic_keys': dynamic_keys,
+	    'confirm_keys': confirm_keys,
+	    'none': none,
+    }
+
+    def __init__(self, config, auth=None):
+	"""
+	Initializer.  Parses a configuration if one is given.
+	"""
+
+	self.debug = config.getboolean("allocate", "debug", False)
+	self.wap = config.get('allocate', 'wap', '/usr/testbed/sbin/wap')
+	self.newproj = config.get('allocate', 'newproj',
+		'/usr/testbed/sbin/newproj')
+	self.mkproj = config.get('allocate', 'mkproj', 
+		'/usr/testbed/sbin/mkproj')
+	self.rmproj = config.get('allocate', 'rmproj',
+		'/usr/testbed/sbin/rmproj')
+	self.rmuser = config.get('allocate', 'rmuser',
+		'/usr/testbed/sbin/rmuser')
+	self.newuser = config.get('allocate', 'newuser',
+		'/usr/testbed/sbin/newuser')
+	self.addpubkey = config.get('allocate', 'addpubkey', 
+		'/usr/testbed/sbin/addpubkey')
+	self.grantnodetype = config.get('allocate', 'grantnodetype', 
+		'/usr/testbed/sbin/grantnodetype')
+	self.confirmkey = config.get('allocate', 'confirmkey', 
+		'/usr/testbed/sbin/taddpubkey')
+	self.user_to_project=config.get("allocate", 'user_to_project',
+		'/usr/local/bin/user_to_project.py')
+	self.allocation_level = config.get("allocate", "allocation_level", 
+		"none")
+	self.log = logging.getLogger("fedd.allocate.local")
+	set_log_level(config, "allocate", self.log)
+
+	if auth:
+	    self.auth = auth
+	else:
+	    auth = authorizer()
+	    log.warn("[allocate] No authorizer passed in, using local one")
+
+	try:
+	    self.allocation_level = \
+		    self.levels[self.allocation_level.strip().lower()]
+	except KeyError:
+	    self.log.error("Bad allocation_level %s.  Defaulting to none" % \
+		    self.allocation_error)
+	    self.allocation_level = self.none
+
+	self.state = { 
+		'keys': set(), 
+		'types': set(),
+		'projects': set(),
+		'users': set(),
+		}
+	self.state_filename = config.get('allocate', 'allocation_state')
+	self.state_lock = threading.Lock()
+	self.read_state()
+
+	access_db = config.get("allocate", "accessdb")
+	if access_db:
+	    try:
+		read_simple_accessdb(access_db, self.auth, 'allocate')
+	    except EnvironmentError, e:
+		raise service_error(service_error.internal,
+			"Error reading accessDB %s: %s" % (access_db, e))
+	    except ValueError:
+		raise service_error(service_error.internal, "%s" % e)
+
+
+	# Internal services are SOAP only
+	self.soap_services = {\
+		"AllocateProject": soap_handler("AllocateProject", 
+		    self.dynamic_project),
+		"StaticProject": soap_handler("StaticProject", 
+		    self.static_project),
+		"ReleaseProject": soap_handler("ReleaseProject", 
+		    self.release_project),
+		}
+	self.xmlrpc_services = { }
+
+    def read_state(self):
+	"""
+	Read a new copy of access state.  Old state is overwritten.
+
+	State format is a simple pickling of the state dictionary.
+	"""
+	if self.state_filename:
+	    try:
+		f = open(self.state_filename, "r")
+		self.state = pickle.load(f)
+		self.log.debug("[allocation]: Read state from %s" % \
+			self.state_filename)
+	    except EnvironmentError, e:
+		self.log.warning(("[allocation]: No saved state: " +\
+			"Can't open %s: %s") % (self.state_filename, e))
+	    except EOFError, e:
+		self.log.warning(("[allocation]: " +\
+			"Empty or damaged state file: %s:") % \
+			self.state_filename)
+	    except pickle.UnpicklingError, e:
+		self.log.warning(("[allocation]: No saved state: " + \
+			"Unpickling failed: %s") % e)
+	    # These should all be in the picked representation, but make sure
+	    if not self.state.has_key('keys'): self.state['keys'] = set()
+	    if not self.state.has_key('types'): self.state['types'] = set()
+	    if not self.state.has_key('projects'):
+		self.state['projects'] = set()
+	    if not self.state.has_key('users'): self.state['users'] = set()
+
+    def write_state(self):
+	if self.state_filename:
+	    try:
+		f = open(self.state_filename, 'w')
+		pickle.dump(self.state, f)
+	    except EnvironmentError, e:
+		self.log.error("Can't write file %s: %s" % \
+			(self.state_filename, e))
+	    except pickle.PicklingError, e:
+		self.log.error("Pickling problem: %s" % e)
+	    except TypeError, e:
+		self.log.error("Pickling problem (TypeError): %s" % e)
+
+
+    def random_string(self, s, n=3):
+	"""Append n random ASCII characters to s and return the string"""
+	rv = s
+	for i in range(0,n):
+	    rv += random.choice(string.ascii_letters)
+	return rv
+
+    def write_attr_xml(self, file, root, lines):
+	"""
+	Write an emulab config file for a dynamic project.
+
+	Format is <root><attribute name=lines[0]>lines[1]</attribute></root>
+	"""
+	# Convert a pair to an attribute line
+	out_attr = lambda a,v : \
+		'<attribute name="%s"><value>%s</value></attribute>' % (a, v)
+
+	f = os.fdopen(file, "w")
+	f.write("<%s>\n" % root)
+	f.write("\n".join([out_attr(*l) for l in lines]))
+	f.write("</%s>\n" % root)
+	f.close()
+
+    def run_cmd(self, cmd, log_prefix='allocate'):
+	"""
+	Run the command passed in.  Cmd is a list containing the words of the
+	command.  Return the exit value from the subprocess - that is 0 on
+	success.  On an error running the command -  python or OS error, raise
+	a service exception.
+	"""
+	try:
+	    dnull = open("/dev/null", "w")
+	except EnvironmentError:
+	    self.log.warn("[run_cmd]: failed to open /dev/null for redirect")
+	    dnull = None
+
+	self.log.debug("[%s]: %s" % (log_prefix, ' '.join(cmd)))
+	if not self.debug:
+	    try:
+		return subprocess.call(cmd, stdout=dnull, stderr=dnull)
+	    except OSError, e:
+		raise service_error(service_error.internal,
+			"Static project subprocess creation error "+ \
+				"[%s] (%s)" %  (cmd[0], e.strerror))
+	else:
+	    return 0
+
+    def confirm_key(self, user, key):
+	"""
+	Call run_cmd to comfirm the key.  Return a boolean rather
+	than the subprocess code.
+	"""
+	return self.run_cmd((self.wap, self.confirmkey, '-C',  
+	    '-u', user, '-k', key)) ==0
+
+    def add_key(self, user, key):
+	"""
+	Call run_cmd to add the key.  Return a boolean rather
+	than the subprocess code.
+	"""
+	return self.run_cmd((self.wap, self.addpubkey, '-u', user,
+	    '-k', key)) == 0
+
+    def remove_key(self, user, key):
+	"""
+	Call run_cmd to remove the key.  Return a boolean rather
+	than the subprocess code.
+	"""
+	return self.run_cmd((self.wap, self.addpubkey, '-R', '-u', user, 
+	    '-k', key)) == 0
+
+    def confirm_access(self, project, type):
+	"""
+	Call run_cmd to comfirm the key.  Return a boolean rather
+	than the subprocess code.
+	"""
+	return self.run_cmd((self.wap, self.grantnodetype, '-C',  
+	    '-p', project, type)) ==0
+
+    def add_access(self, project, type):
+	"""
+	Call run_cmd to add the key.  Return a boolean rather
+	than the subprocess code.
+	"""
+	return self.run_cmd((self.wap, self.grantnodetype, 
+	    '-p', project, type)) == 0
+
+    def remove_access(self, project, type):
+	"""
+	Call run_cmd to remove the key.  Return a boolean rather
+	than the subprocess code.
+	"""
+
+	return self.run_cmd((self.wap, self.grantnodetype, '-R', 
+	    '-p', project, type)) == 0
+
+    def add_project(self, project, projfile):
+	"""
+	Create a project using run_cmd.  This is two steps, and assumes that
+	the relevant XML files are in place and correct.  Make the return value
+	boolean.  Note that if a new user is specified in the XML, that user is
+	created on success.
+	"""
+
+	if self.run_cmd((self.wap, self.newproj, projfile)) == 0:
+	    return self.run_cmd((self.wap, self.mkproj, project)) ==0
+	else:
+	    return False
+
+    def remove_project(self, project):
+	"""
+	Call run_cmd to remove the project.  Make the return value boolean.
+	"""
+
+	return self.run_cmd(self.wap, self.rmproj, project) == 0
+
+   
+    def add_user(self, name, param_file, project):
+	"""
+	Create a user and link them to the given project.  Similar to
+	add_project, this requires a two step approach.  Returns True on success
+	False on failure.
+	"""
+
+	if self.run_cmd((self.wap, self.newuser, param_file)) == 0:
+	   return self.run_cmd((self.wap, self.user_to_project,
+	       user, project)) == 0
+	else:
+	   return False
+
+    def remove_user(self, user):
+	"""
+	Call run_cmd to remove the user.  Make the return value boolean.
+	"""
+
+	return self.run_cmd(self.wap, self.rmuser, user) == 0
+
+
+
+    def dynamic_project(self, req, fedid=None):
+	"""
+	Create a dynamic project with ssh access
+
+	Req includes the project and resources as a dictionary
+	"""
+
+	# Internal calls do not have a fedid parameter (i.e., local calls on
+	# behalf of already vetted fedids)
+	if fedid and not self.auth.check_attribute(fedid, "allocate"):
+	    self.log.debug("[allocate] Access denied (%s)" % fedid)
+	    raise service_error(service_error.access, "Access Denied")
+
+	if self.allocation_level < self.dynamic_projects:
+	    raise service_error(service_error.access, 
+		    "[dynamic_project] dynamic project allocation not " + \
+			    "permitted: check allocation level")
+	# tempfiles for the parameter files
+	cuf, create_userfile = tempfile.mkstemp(prefix="usr", suffix=".xml",
+		dir="/tmp")
+	suf, service_userfile = tempfile.mkstemp(prefix="usr", suffix=".xml",
+		dir="/tmp")
+	pf, projfile = tempfile.mkstemp(prefix="proj", suffix=".xml",
+		dir="/tmp")
+
+	if req.has_key('AllocateProjectRequestBody'):
+	    proj = req['AllocateProjectRequestBody'].get('project', None)
+	    if not proj:
+		raise service_error(service_error.req, 
+			"Badly formed allocation request")
+	    resources = req['AllocateProjectRequestBody'].get('resources', { })
+	else:
+	    raise service_error(service_error.req, 
+		    "Badly formed allocation request")
+	# Take the first user and ssh key
+	name = proj.get('name', None) or self.random_string("proj",4)
+	user = proj.get('user', [])
+
+	uname = { }
+	ssh = { }
+	for u in user:
+	    role = u.get('role', None)
+	    if not role: continue
+	    if u.has_key('userID'):
+		uid = u['userID']
+		uname[role] = uid.get('localname', None) or \
+			uid.get('kerberosUsername', None) or \
+			uid.get('uri', None)
+		if uname[role] == None:
+		    raise service_error(service_error.req, "No ID for user")
+	    else:
+		uname[role] = self.random_string("user", 3)
+
+	    access = u.get('access', None)
+	    if access:
+		# XXX collect and call addpubkey later, for now use first one.
+		for a in access:
+		    ssh[role] = a.get('sshPubkey', None)
+		    if ssh: break
+		else:
+		    raise service_error(service_error.req,
+			    "No SSH key for user %s" % uname[role])
+	    else:
+		raise service_error(service_error.req,
+			"No access mechanisms for for user %s" % uname[role])
+
+	if not (uname.has_key('experimentCreation') and \
+		uname.has_key('serviceAccess')):
+	    raise service_error(service_error.req,
+		    "Must specify both user roles")
+	
+
+	create_user_fields = [
+		("name", "Federation User %s" % uname['experimentCreation']),
+		("email", "%s-fed@isi.deterlab.net" % \
+			uname['experimentCreation']),
+		("password", self.random_string("", 8)),
+		("login", uname['experimentCreation']),
+		("address", "4676 Admiralty"),
+		("city", "Marina del Rey"),
+		("state", "CA"),
+		("zip", "90292"),
+		("country", "USA"),
+		("phone", "310-448-9190"),
+		("title", "None"),
+		("affiliation", "USC/ISI"),
+		("pubkey", ssh['experimentCreation'])
+	]
+
+	service_user_fields = [
+		("name", "Federation User %s" % uname['serviceAccess']),
+		("email", "%s-fed@isi.deterlab.net" % uname['serviceAccess']),
+		("password", self.random_string("", 8)),
+		("login", uname['serviceAccess']),
+		("address", "4676 Admiralty"),
+		("city", "Marina del Rey"),
+		("state", "CA"),
+		("zip", "90292"),
+		("country", "USA"),
+		("phone", "310-448-9190"),
+		("title", "None"),
+		("affiliation", "USC/ISI"),
+		("pubkey", ssh['serviceAccess'])
+	]
+
+	proj_fields = [
+		("name", name),
+		("short description", "dynamic federated project"),
+		("URL", "http://www.isi.edu/~faber"),
+		("funders", "USC/USU"),
+		("long description", "Federation access control"),
+		("public", "1"),
+		("num_pcs", "100"),
+		("linkedtous", "1"),
+		("newuser_xml", create_userfile)
+	]
+	
+
+
+	added_projects = [ ]
+	added_users = [ ]
+	added_types = [ ]
+
+	self.state_lock.acquire()
+	try:
+	    # Write out the files
+	    self.write_attr_xml(cuf, "user", create_user_fields)
+	    self.write_attr_xml(suf, "user", service_user_fields)
+	    self.write_attr_xml(pf, "project", proj_fields)
+	    try:
+		if self.add_project(name, projfile):
+		    # add_project adds a user as well in this case
+		    added_projects.append(name)
+		    added_users.append(uname['createExperiment'])
+		    self.state['projects'].add(name)
+		    self.state['users'].add(uname['createExperiment'])
+
+		    if self.add_user(uname['serviceAccess'], 
+			    service_userfile, name):
+			added_users.append(uname['serviceAccess'])
+			self.state['users'].add(uname['serviceAccess'])
+		    else:
+			raise service_error("Unable to create user %s" % \
+				uname['serviceAccess'])
+		else:
+		    raise service_error("Unable to create project/user %s/%s" % \
+			    (name, uname['experimentCreation']))
+
+		nodes = resources.get('node', [])
+		# Grant access to restricted resources.  This is simpler than
+		# the corresponding loop from static_project because this is a
+		# clean slate.
+		for nt in [ h for n in nodes\
+			if n.has_key('hardware')\
+			    for h in n['hardware'] ] :
+		    if self.add_access(name, nt):
+			self.state['types'].add((name, nt))
+			added_types.append((name, nt))
+		    else:
+			raise service_error(service_error.internal,
+				"Failed to add access for %s to %s"\
+					% (name, nt))
+	    except service_error, e:
+		# Something failed.  Back out the partial allocation as
+		# completely as possible and re-raise the error.
+		for p, t in added_types:
+		    self.state['types'].discard((p, t))
+		    try:
+			self.remove_access(p, t)
+		    except service_error:
+			pass
+		for u in added_users:
+		    self.state['users'].discard(u)
+		    try:
+			self.remove_user(u)
+		    except service_error:
+			pass
+
+		for p in added_projects:
+		    self.state['projects'].discard(p)
+		    try:
+			self.remove_project(p)
+		    except service_error:
+			pass
+		self.state_lock.release()
+		raise e
+	finally:
+	    # Clean up tempfiles
+	    os.unlink(create_userfile)
+	    os.unlink(service_userfile)
+	    os.unlink(projfile)
+
+	rv = {\
+	    'project': {\
+		'name': { 'localname': name }, 
+		'user' : [\
+		    {\
+			'userID': { 'localname' : uname['experimentCreation'] },
+			'access': [ {'sshPubkey': ssh['experimentCreation'] } ],
+			'role': 'experimentCreation',
+		    }, \
+		    {\
+			'userID': { 'localname' : uname['serviceAccess'] },
+			'access': [ { 'sshPubkey' : ssh['serviceAccess'] } ], 
+			'role': 'serviceAccess',
+		    } \
+		]\
+	    }\
+	}
+	return rv
+
+    def static_project(self, req, fedid=None):
+	"""
+	Be certain that the local project in the request has access to the
+	proper resources and users have correct keys.  Add them if necessary.
+	"""
+	# Internal calls do not have a fedid parameter (i.e., local calls on
+	# behalf of already vetted fedids)
+	if fedid and not self.auth.check_attribute(fedid, "allocate"):
+	    self.log.debug("[allocate] Access denied (%s)" % fedid)
+	    raise service_error(service_error.access, "Access Denied")
+	# While we should be more careful about this, for the short term, add
+	# the keys to the specified users.
+
+	try:
+	    users = req['StaticProjectRequestBody']['project']['user']
+	    pname = req['StaticProjectRequestBody']['project']\
+		    ['name']['localname']
+	    resources = req['StaticProjectRequestBody'].get('resources', { })
+	except KeyError:
+	    raise service_error(service_error.req, "Badly formed request")
+
+	added_keys = [ ]
+	added_types = [ ]
+	# Keep track of changes made to the system
+	self.state_lock.acquire()
+
+	try:
+	    for u in users:
+		try:
+		    name = u['userID']['localname']
+		except KeyError:
+		    raise service_error(service_error.req, "Badly formed user")
+		for sk in [ k['sshPubkey'] for k in u.get('access', []) \
+			if k.has_key('sshPubkey')]:
+		    if self.allocation_level >=self.confirm_keys:
+			key_ok = self.confirm_key(name, sk)
+			if not key_ok:
+			    if self.allocation_level >= self.dynamic_keys:
+				if self.add_key(name, sk):
+				    self.state['keys'].add((name, sk))
+				    added_keys.append((name, sk))
+				else:
+				    raise service_error(service_error.internal,
+					    "Failed to add key for %s" % name)
+			    else:
+				raise service_error(service_error.internal,
+					"Failed to confirm key for %s" % name)
+		    else:
+			self.log.warning("[static_project] no checking of " + \
+			    "static keys")
+
+	    # Grant access to any resources in the request.  The
+	    # list comprehension pulls out the hardware types in the node
+	    # entries in the resources list.  The access module knows to
+	    # only send resources that are restricted and needed by the
+	    # project.
+	    nodes = resources.get('node', [])
+	    for nt in [ h for n in nodes\
+		    if n.has_key('hardware')\
+			for h in n['hardware'] ] :
+		if self.allocation_level >= self.confirm_keys:
+		    access_ok = self.confirm_access(pname, nt)
+		    if not access_ok:
+			if self.allocation_level >= self.dynamic_keys:
+			    if self.add_access(pname, nt):
+				self.state['types'].add((pname, nt))
+				added_types.append((pname, nt))
+			    else:
+				raise service_error(service_error.internal,
+					"Failed to add access for %s to %s"\
+						% (pname, nt))
+			else:
+			    raise service_error(service_error.internal,
+				    "Failed to confirm access for %s to %s"\
+					    % (pname, nt))
+		else:
+		    self.log.warning("[static_project] no checking of " + \
+			"node access")
+	except service_error, e:
+	    # Do our best to clean up partial allocation and reraise the
+	    # error.  Do our best to make sure that both allocation state and
+	    # testbed state is restored.
+	    for u, k in added_keys:
+		self.state['keys'].discard((u, k))
+		try:
+		    self.remove_key(u, k)
+		except service_error:
+		    pass
+	    for p, t in added_types:
+		self.state['types'].discard((p, t))
+		try:
+		    self.remove_access(p, t)
+		except service_error:
+		    pass
+	    self.state_lock.release()
+	    raise e
+	# All is well, save state and release the lock
+	self.write_state()
+	self.state_lock.release()
+	# return { 'project': req['StaticProjectRequestBody']['project']}
+	return req['StaticProjectRequestBody']
+
+    def release_project(self, req, fedid=None):
+	"""
+	Remove user keys from users and delete dynamic projects.
+
+	Only keys this service created are deleted and there are
+	similar protections for projects.
+	"""
+	# Internal calls do not have a fedid parameter (i.e., local calls on
+	# behalf of already vetted fedids)
+	if fedid and not self.auth.check_attribute(fedid, "allocate"):
+	    self.log.debug("[allocate] Access denied (%s)" % fedid)
+	    raise service_error(service_error.access, "Access Denied")
+
+	pname = None
+	users = []
+	nodes = [ ]
+
+	try:
+	    if req['ReleaseProjectRequestBody']['project'].has_key('name'):
+		pname = req['ReleaseProjectRequestBody']['project']\
+			['name']['localname']
+	    if req['ReleaseProjectRequestBody']['project'].has_key('user'):
+		users = req['ReleaseProjectRequestBody']['project']['user']
+	    if req['ReleaseProjectRequestBody'].has_key('resources'):
+		nodes = req['ReleaseProjectRequestBody']\
+			['resources'].get('node', [])
+	except KeyError:
+	    raise service_error(service_error.req, "Badly formed request")
+
+	if nodes and not pname:
+	    raise service_error(service_error.req, 
+		    "Badly formed request (nodes without project)")
+
+	self.state_lock.acquire()
+	try:
+	    for nt in [ h for n in nodes if n.has_key('hardware')\
+		    for h in n['hardware'] ] :
+		if (pname, nt ) in self.state['types']:
+		    self.remove_access(pname, nt)
+		    self.state['types'].discard((pname, nt))
+
+	    for u in users:
+		try:
+		    name = u['userID']['localname']
+		except KeyError:
+		    raise service_error(service_error.req, "Badly formed user")
+		if name in self.state['users']:
+		    # If we created this user, discard the user, keys and all
+		    self.remove_user(name)
+		    self.state['users'].discard(name)
+		else:
+		    # If not, just strip any keys we added
+		    for sk in [ k['sshPubkey'] for k in u.get('access', []) \
+			    if k.has_key('sshPubkey')]:
+			if (name, sk) in self.state['keys']:
+			    self.remove_key(name, sk)
+			    self.state['keys'].discard((name, sk))
+	    if pname in self.state['projects']:
+		self.remove_project(pname)
+		self.state['projects'].discard(pname)
+
+	except service_error, e:
+	    self.write_state()
+	    self.state_lock.release()
+	    raise e
+	self.write_state()
+	self.state_lock.release()
+
+	return { 'project': req['ReleaseProjectRequestBody']['project']}
+
+class allocate_project_remote:
+    """
+    Allocate projects on a remote machine using the internal SOAP interface
+    """
+    class proxy(service_caller):
+	"""
+	This class is a proxy functor (callable) that has the same signature as
+	a function called by soap_handler or xmlrpc_handler, but that used the
+	service_caller class to call the function remotely.
+	"""
+
+	def __init__(self, url, cert_file, cert_pwd, trusted_certs, auth, 
+		method):
+	    service_caller.__init__(self, method)
+	    self.url = url
+	    self.cert_file = cert_file
+	    self.cert_pwd = cert_pwd
+	    self.trusted_certs = trusted_certs
+	    self.request_body__name = "%sRequestBody" % method
+	    self.resp_name = "%sResponseBody" % method
+	    self.auth = auth
+	    # Calling the proxy object directly invokes the proxy_call method,
+	    # not the service_call method.
+	    self.__call__ = self.proxy_call
+	    
+
+	# Define the proxy, NB, the parameters to make_proxy are visible to the
+	# definition of proxy.
+	def proxy_call(self, req, fid=None):
+	    """
+	    Send req on to a remote project instantiator.
+
+	    Req is just the message to be sent.  This function re-wraps it.
+	    It also rethrows any faults.
+	    """
+
+	    if req.has_key(self.request_body_name):
+		req = req[self.request_body_name]
+	    else:
+		raise service_error(service_error.req, "Bad formated request");
+
+	    try:
+		r = self.call_service(self.url, req, self.cert_file,
+			self.cert_pwd, self.trusted_certs)
+	    except service_error, e:
+		if e.code == service_error.connect:
+		    raise service_error(service_error.internal, 
+			    "Cannot connect to internal service: (%d) %s" % \
+				    (e.code, e.desc))
+		else: raise
+	    if r.has_key(self.resp_name):
+		return r[self.resp_name]
+	    else:
+		raise service_error(service_error.protocol, 
+			"Bad proxy response")
+
+    # back to defining the allocate_project_remote class
+    def __init__(self, config, auth=None):
+	"""
+	Initializer.  Parses a configuration if one is given.
+	"""
+
+	self.debug = config.get("allocate", "debug", False)
+	self.url = config.get("allocate", "uri", "")
+
+	# Keep cert file and password coming from the same source
+	self.cert_file = config.get("allocate", "cert_file", None)
+	if self.cert_file:
+	    self.cert_pwd = config.get("allocate", "cert_pwd", None)
+	else:
+	    self.cert_file = config.get("globals", "cert_file", None)
+	    self.cert_pwd = config.get("globals", "cert_pwd", None)
+
+	self.trusted_certs = config.get("allocate", "trusted_certs", None) or \
+		config.get("globals", "trusted_certs")
+
+	self.soap_services = { }
+	self.xmlrpc_services = { }
+	self.log = logging.getLogger("fedd.allocate.remote")
+	set_log_level(config, "allocate", self.log)
+
+	if auth:
+	    self.auth = auth
+	else:
+	    auth = authorizer()
+	    log.warn("[allocate] No authorizer passed in, using local one")
+
+	# The specializations of the proxy functions
+	self.dynamic_project = self.proxy(self.url, self.cert_file, 
+		self.cert_pwd, self.trusted_certs, self.auth, 
+		"AllocateProject")
+	self.static_project = self.proxy(self.url, self.cert_file, 
+		self.cert_pwd, self.trusted_certs, self.auth, 
+		"StaticProject")
+	self.release_project = self.proxy(self.url, self.cert_file,
+		self.cert_pwd, self.trusted_certs, self.auth, 
+		"ReleaseProject")
+
Index: fedd/Makefile/federation/asn1_raw.py
===================================================================
--- fedd/Makefile/federation/asn1_raw.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/asn1_raw.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,160 @@
+# Modified from an example in the Python ASN.1 package to read certificate
+# files and extract key bits.  More recent versions of m2crypto support this
+# more directly.
+
+import sys, string, base64
+from pyasn1.type import tag,namedtype,namedval,univ,constraint,char,useful
+from pyasn1.codec.der import decoder, encoder
+from pyasn1 import error
+
+from M2Crypto import EVP
+
+# Would be autogenerated from ASN.1 source by a ASN.1 parser
+# X.509 spec (rfc2459)
+
+MAX = 64  # XXX ?
+
+class DirectoryString(univ.Choice):
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('teletexString', char.TeletexString().subtype(subtypeSpec=constraint.ValueSizeConstraint(1, MAX))),
+        namedtype.NamedType('printableString', char.PrintableString().subtype(subtypeSpec=constraint.ValueSizeConstraint(1, MAX))),
+        namedtype.NamedType('universalString', char.UniversalString().subtype(subtypeSpec=constraint.ValueSizeConstraint(1, MAX))),
+        namedtype.NamedType('utf8String', char.UTF8String().subtype(subtypeSpec=constraint.ValueSizeConstraint(1, MAX))),
+        namedtype.NamedType('bmpString', char.BMPString().subtype(subtypeSpec=constraint.ValueSizeConstraint(1, MAX))),
+        namedtype.NamedType('ia5String', char.IA5String().subtype(subtypeSpec=constraint.ValueSizeConstraint(1, MAX))) # hm, this should not be here!? XXX
+        )
+
+class AttributeValue(DirectoryString): pass
+
+class AttributeType(univ.ObjectIdentifier): pass
+
+class AttributeTypeAndValue(univ.Sequence):
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('type', AttributeType()),
+        namedtype.NamedType('value', AttributeValue())
+        )
+
+class RelativeDistinguishedName(univ.SetOf):
+    componentType = AttributeTypeAndValue()
+
+class RDNSequence(univ.SequenceOf):
+    componentType = RelativeDistinguishedName()
+
+class Name(univ.Choice):
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('', RDNSequence())
+        )
+                          
+class AlgorithmIdentifier(univ.Sequence):
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('algorithm', univ.ObjectIdentifier()),
+        namedtype.OptionalNamedType('parameters', univ.Null())
+        # XXX syntax screwed?
+#        namedtype.OptionalNamedType('parameters', univ.ObjectIdentifier())
+        )
+
+class Extension(univ.Sequence):
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('extnID', univ.ObjectIdentifier()),
+        namedtype.DefaultedNamedType('critical', univ.Boolean('False')),
+        namedtype.NamedType('extnValue', univ.OctetString())
+        )
+
+class Extensions(univ.SequenceOf):
+    componentType = Extension()
+    sizeSpec = univ.SequenceOf.sizeSpec + constraint.ValueSizeConstraint(1, MAX)
+
+class SubjectPublicKeyInfo(univ.Sequence):
+     componentType = namedtype.NamedTypes(
+         namedtype.NamedType('algorithm', AlgorithmIdentifier()),
+         namedtype.NamedType('subjectPublicKey', univ.BitString())
+         )
+
+class UniqueIdentifier(univ.BitString): pass
+
+class Time(univ.Choice):
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('utcTime', useful.UTCTime()),
+        namedtype.NamedType('generalTime', useful.GeneralizedTime())
+        )
+    
+class Validity(univ.Sequence):
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('notBefore', Time()),
+        namedtype.NamedType('notAfter', Time())
+        )
+
+class CertificateSerialNumber(univ.Integer): pass
+
+class Version(univ.Integer):
+    namedValues = namedval.NamedValues(
+        ('v1', 0), ('v2', 1), ('v3', 2)
+        )
+
+class TBSCertificate(univ.Sequence):
+    componentType = namedtype.NamedTypes(
+        namedtype.DefaultedNamedType('version', Version('v1', tagSet=Version.tagSet.tagExplicitly(tag.Tag(tag.tagClassContext, tag.tagFormatSimple, 0)))),
+        namedtype.NamedType('serialNumber', CertificateSerialNumber()),
+        namedtype.NamedType('signature', AlgorithmIdentifier()),
+        namedtype.NamedType('issuer', Name()),
+        namedtype.NamedType('validity', Validity()),
+        namedtype.NamedType('subject', Name()),
+        namedtype.NamedType('subjectPublicKeyInfo', SubjectPublicKeyInfo()),
+        namedtype.OptionalNamedType('issuerUniqueID', UniqueIdentifier().subtype(implicitTag=tag.Tag(tag.tagClassContext, tag.tagFormatSimple, 1))),
+        namedtype.OptionalNamedType('subjectUniqueID', UniqueIdentifier().subtype(implicitTag=tag.Tag(tag.tagClassContext, tag.tagFormatSimple, 2))),
+        namedtype.OptionalNamedType('extensions', Extensions().subtype(explicitTag=tag.Tag(tag.tagClassContext, tag.tagFormatSimple, 3)))
+        )
+
+class Certificate(univ.Sequence):
+    componentType = namedtype.NamedTypes(
+        namedtype.NamedType('tbsCertificate', TBSCertificate()),
+        namedtype.NamedType('signatureAlgorithm', AlgorithmIdentifier()),
+        namedtype.NamedType('signatureValue', univ.BitString())
+        )
+
+# end of ASN.1 data structures
+
+
+def get_bits(substrate):
+    certType = Certificate()
+    cert = decoder.decode(substrate, asn1Spec=certType)[0]
+    n = cert
+    for name in ('tbsCertificate', 'subjectPublicKeyInfo', 
+	    'subjectPublicKey'):
+	n = n.getComponentByName(name)
+	if n == None: raise RuntimeException("Bad cert")
+
+    b = []
+    for i in range(0, len(n), 8):
+	v = 0
+	for j in range(0, 8):
+	    v = (v << 1) + n[i+j]
+	b.append(v)
+
+    return str().join([chr(x) for x in b])
+
+
+def get_key_bits_from_file(fname):
+    stSpam, stHam, stDump = 0, 1, 2
+    state = stSpam
+
+    f = open(fname)
+
+    for certLine in f:
+	certLine = string.strip(certLine)
+	if state == stSpam:
+	    if certLine == '-----BEGIN CERTIFICATE-----':
+		substrate = ""
+		state = stHam
+		continue
+	if state == stHam:
+	    if certLine == '-----END CERTIFICATE-----':
+		state = stDump
+	    else:
+		substrate = substrate + base64.b64decode(certLine)
+	if state == stDump:
+	    f.close()
+	    return get_bits(substrate)
+
+def get_key_bits_from_cert(cert):
+    return get_bits(cert.as_der())
Index: fedd/Makefile/federation/authorizer.py
===================================================================
--- fedd/Makefile/federation/authorizer.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/authorizer.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,242 @@
+#/usr/local/bin/python
+
+from fedid import fedid
+from remote_service import service_caller
+from abac_remote_service import abac_service_caller
+from service_error import service_error
+
+import sys
+
+class authorizer_base:
+    """
+    Classes based on this one keep track of authorization attributes for the
+    various modules running.  This base class holds some utility functions that
+    they all potentially use.
+    """
+
+    # general error exception for badly formed names.  
+    class bad_name(RuntimeError): pass
+
+    @staticmethod
+    def auth_name(name):
+	"""
+	Helper to convert a non-unicode local name to a unicode string.  Mixed
+	representations can needlessly confuse the authorizer.
+	"""
+	if isinstance(name, basestring):
+	    if not isinstance(name, unicode): return unicode(name)
+	    else: return name
+	else: return name
+
+    @staticmethod
+    def valid_name(name):
+	"""
+	Ensure that the given name is valid.  A valid name can either be a
+	triple of strings and fedids representing one of our generalized Emulab
+	names or a single fedid.  Compound names can include wildcards (None)
+	and must anchor to a fedid at their highest level (unless they're all
+	None)
+
+	This either returns True or throws an exception.  More an assertion
+	than a function.
+	"""
+	if isinstance(name, tuple) and len(name) == 3:
+	    for n in name:
+		if n: 
+		    if not (isinstance(n, basestring) or isinstance(n, fedid)):
+			raise authorizer_base.bad_name(
+				"names must be either a triple or a fedid")
+	    for n in name: 
+		if n:
+		    if isinstance(n, fedid):
+			return True
+		    else:
+			raise authorizer_base.bad_name(
+				"Compound names must be " + \
+				"rooted in fedids: %s" % str(name))
+
+	    return True
+	elif isinstance(name, fedid):
+	    return True
+	else:
+	    raise authorizer_base.bad_name(
+		    "Names must be a triple or a fedid (%s)" % name)
+
+
+class authorizer(authorizer_base):
+    """
+    This class keeps track of authorization attributes for the various modules
+    running.  When it gets smarter it will be the basis for a real
+    attribute-based authentication system.
+    """
+    def __init__(self, def_attr="testbed"):
+	self.attrs = { }
+	self.globals=set()
+
+    def set_attribute(self, name, attr):
+	"""
+	Attach attr to name.  Multiple attrs can be attached.
+	"""
+	self.valid_name(name)
+	if isinstance(name, tuple):
+	    aname = tuple([ self.auth_name(n) for n in name])
+	else:
+	    aname = self.auth_name(name)
+
+	if not self.attrs.has_key(aname):
+	    self.attrs[aname] = set()
+	self.attrs[aname].add(attr)
+
+    def unset_attribute(self, name, attr):
+	"""
+	Remove an attribute from name
+	"""
+	self.valid_name(name)
+	if isinstance(name, tuple):
+	    aname = tuple([ self.auth_name(n) for n in name])
+	else:
+	    aname = self.auth_name(name)
+
+	attrs = self.attrs.get(aname, None)
+	if attrs: attrs.discard(attr)
+
+    def check_attribute(self, name, attr):
+	"""
+	Return True if name has attr (or if attr is global).  Tuple names match
+	any tuple name that matches all names present and has None entries in
+	other fileds.  For tuple names True implies that there is a matching
+	tuple name with the attribute.
+	"""
+	def tup(tup, i, p):
+	    mask = 1 << i
+	    if p & mask : return authorizer.auth_name(tup[i])
+	    else: return None
+
+	self.valid_name(name)
+	if attr in self.globals:
+	    return True
+
+	if isinstance(name, tuple):
+	    for p in range(0,8):
+		lookup = ( tup(name, 0, p), tup(name,1, p), tup(name,2,p))
+		if self.attrs.has_key(lookup):
+		    if attr in self.attrs[lookup]:
+			return True
+	else:
+	    return  attr in self.attrs.get(self.auth_name(name), set())
+
+    def set_global_attribute(self, attr):
+	"""
+	Set a global attribute.  All names, even those otherwise unknown to the
+	authorizer have this attribute.
+	"""
+	self.globals.add(attr)
+
+    def unset_global_attribute(self, attr):
+	"""
+	Remove a global attribute
+	"""
+
+	self.globals.discard(attr)
+
+    def __str__(self):
+	rv = ""
+	rv += "attrs %s\n" % self.attrs
+	rv += "globals %s" % self.globals
+	return rv
+
+class abac_authorizer(authorizer_base):
+    """
+    Use the ABAC authorization system to make attribute decisions.
+    """
+
+    def __init__(self, url=None, cert_file=None, cert_pwd=None,
+	    trusted_certs=None, me=None):
+	self.call_CreateContext = abac_service_caller('CreateContext')
+	self.call_CredentialUpdate = abac_service_caller('CredentialUpdate')
+	self.call_Access = abac_service_caller('Access')
+	self.globals = set()
+
+	self.url = url
+	self.cert_file = cert_file
+	self.cert_pwd = cert_pwd
+	self.trusted_certs = trusted_certs
+	self.me = me
+	req = { 'context': { 'self': me.get_hexstr()} }
+	# May throw a service error
+	resp = self.call_CreateContext(self.url, req, self.cert_file,
+		self.cert_pwd, self.trusted_certs, tracefile=sys.stderr)
+
+	if resp.has_key('CreateContextResponseBody'):
+	    resp =  resp['CreateContextResponseBody']
+	    if resp.has_key('contextID'):
+		self.contextID = resp['contextID']
+	    else:
+		raise service_error(service_error.internal, 
+			"No context ID for the new authorization context")
+	else:
+	    raise service_error(service_error.internal, 
+		    "Bad response to creating service context")
+
+    def set_attribute(self, name, attr):
+	self.valid_name(name)
+	if isinstance(name, tuple):
+	    if not isinstance(name[0], fedid):
+		raise service_error(service_error.internal, 
+			"Triple not anchored in testbed")
+	    if not name[1] and not name[2]:
+		raise service_error(service_error.internal,
+			"Anonymous access not yet permitted")
+
+	    preconditions = "^".join(
+		    [ "%s.%s" % (name[0], self.auth_name(n)) \
+			    for n in name[1:3] if n ])
+
+	    req = { 
+		    'credential': [ '%s.%s<--%s' % \
+			    ( self.me, attr, preconditions) ],
+		    'context': { 'contextID': self.contextID  }
+		}
+	else:
+	    req = { 
+		    'credential': [ '%s.%s<--%s' % \
+			    ( self.me, attr, self.auth_name(name)) ],
+		    'context': { 'contextID': self.contextID }
+		}
+	print req
+	self.call_CredentialUpdate(self.url, req, self.cert_file, 
+		self.cert_pwd, self.trusted_certs, tracefile=sys.stderr)
+
+    def check_attribute(self, name, attr):
+	self.valid_name(name)
+	if attr in self.globals: return True
+
+	if isinstance(name, tuple):
+	    raise service_error(service_error.internal, 
+		    "Only fedids permitted here")
+	req = {
+		'goal': '%s:%s<<---%s' % (self.me, attr, name),
+		'context': { 'contextID': self.contextID }
+	    }
+	resp = self.call_Access(self.url, req, self.cert_file, self.cert_pwd, 
+		self.trusted_certs)
+	if resp.has_key('AccessResponseBody'):
+	    resp = resp['AccessResponseBody']
+
+	result = resp.get('result', None)
+	return (result and result.lower() == 'true')
+
+    def set_global_attribute(self, attr):
+	"""
+	Set a global attribute.  All names, even those otherwise unknown to the
+	authorizer have this attribute.
+	"""
+	self.globals.add(attr)
+
+    def unset_global_attribute(self, attr):
+	"""
+	Remove a global attribute
+	"""
+
+	self.globals.discard(attr)
+
Index: fedd/Makefile/federation/client_lib.py
===================================================================
--- fedd/Makefile/federation/client_lib.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/client_lib.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,280 @@
+#!/usr/local/bin/python
+
+import sys
+import os
+import pwd
+
+
+from fedid import fedid
+from util import fedd_ssl_context
+from remote_service import service_caller
+from service_error import service_error
+
+from optparse import OptionParser, OptionValueError
+
+
+class client_opts(OptionParser):
+    """
+    Standatd set of options that all clients talking to fedd can probably use.
+    Client code usually specializes this.
+    """
+    def __init__(self):
+        OptionParser.__init__(self, usage="%prog [opts] (--help for details)",
+                version="0.1")
+
+        self.add_option("--cert", action="store", dest="cert",
+                type="string", help="my certificate file")
+        self.add_option( "--debug", action="count", dest="debug", 
+                default=0, help="Set debug.  Repeat for more information")
+        self.add_option("--serialize_only", action="store_true", 
+                dest="serialize_only", default=False,
+                help="Print the SOAP request that would be sent and exit")
+        self.add_option("--trusted", action="store", dest="trusted",
+                type="string", help="Trusted certificates (required)")
+        self.add_option("--url", action="store", dest="url",
+                type="string",default="https://localhost:23235",  
+                help="URL to connect to (default %default)")
+        self.add_option("--transport", action="store", type="choice",
+                choices=("xmlrpc", "soap"), default="soap",
+                help="Transport for request (xmlrpc|soap) (Default: %default)")
+        self.add_option("--trace", action="store_const", dest="tracefile", 
+                const=sys.stderr, help="Print SOAP exchange to stderr")
+
+def exit_with_fault(exc, out=sys.stderr):
+    """
+    Print an error message and exit.  exc is the RPCException that caused the
+    failure.
+    """
+    codestr = ""
+
+    if exc.errstr: codestr = "Error: %s" % exc.errstr
+    else: codestr = ""
+
+    if exc.code:
+	if isinstance(exc.code, int): code = exc.code
+	else: code = -1
+
+	if len(codestr) > 0 : codestr += " (%d)" % code
+	else: codestr = "Error Code: %d" % code
+    else:
+	code = -1
+
+    print>>out, codestr
+    print>>out, "Description: %s" % exc.desc
+    sys.exit(code)
+
+class RPCException(RuntimeError):
+    """
+    An error during the RPC exchange.  It unifies errors from both SOAP and
+    XMLPRC calls.
+    """
+    def __init__(self, desc=None, code=None, errstr=None):
+	RuntimeError.__init__(self)
+	self.desc = desc
+	if isinstance(code, int): self.code = code
+	else: self.code = -1
+	self.errstr = errstr
+
+
+def get_user_cert():
+    cert = os.path.expanduser("~/.ssl/emulab.pem")
+    if not os.access(cert, os.R_OK):
+	cert = None
+    return cert
+
+def wrangle_standard_options(opts):
+    """
+    Look for the certificate to use for the call (check for the standard emulab
+    file and any passed in.  Make sure a present cert file can be read and
+    generate a fedid from it.  Make sure that any trusted cert file can be read
+    and if the debug level is set, set the tracefile attribute as well.  If any
+    of these tests fail, raise a RuntimeError.  Otherwise return the
+    certificate file and the dedid.
+    """
+    if opts.debug > 0: opts.tracefile=sys.stderr
+
+    if opts.trusted:
+	if ( not os.access(opts.trusted, os.R_OK) ) :
+	    raise RuntimeError("Cannot read trusted certificates (%s)" \
+		    % opts.trusted)
+
+    cert = get_user_cert()
+    if opts.cert: cert = opts.cert
+
+    if cert is None:
+	raise RuntimeError("No certificate given (--cert) or found")
+
+    if os.access(cert, os.R_OK):
+	fid = fedid(file=cert)
+    else:
+	raise RuntimeError("Cannot read certificate (%s)" % cert)
+
+    return (cert, fid)
+
+def save_certfile(out_certfile, ea):
+    """
+    if the experiment authority section in ea has a certificate and the
+    out_certfile parameter has a place to put it, save the cert to the file.
+    EnvronnemtError s can come from the file operations.
+    """
+    if out_certfile and ea and 'X509' in ea:
+	f = open(out_certfile, "w")
+	print >>f, ea['X509']
+	f.close()
+
+
+def do_rpc(req_dict, url, transport, cert, trusted, tracefile=None,
+	serialize_only=False, caller=None, responseBody=None):
+    """
+    The work of sending and parsing the RPC as either XMLRPC or SOAP
+    """
+
+    if caller is None: 
+	raise RuntimeError("Must provide caller to do_rpc")
+
+    context = None
+    while context == None:
+	try:
+	    context = fedd_ssl_context(cert, trusted)
+	except Exception, e:
+	    # Yes, doing this on message type is not ideal.  The string
+	    # comes from OpenSSL, so check there is this stops working.
+	    if str(e) == "bad decrypt": 
+		print >>sys.stderr, "Bad Passphrase given."
+	    else: raise
+
+    if transport == "soap":
+	if serialize_only:
+	    print caller.serialize_soap(req_dict) 
+	    return { }
+	else:
+	    try:
+		resp = caller.call_soap_service(url, req_dict, 
+			context=context, tracefile=tracefile)
+	    except service_error, e:
+		raise RPCException(desc=e.desc, code=e.code, 
+			errstr=e.code_string())
+    elif transport == "xmlrpc":
+	if serialize_only:
+	    ser = dumps((req_dict,))
+	    print ser
+	    return { }
+	else:
+	    try:
+		resp = caller.call_xmlrpc_service(url, req_dict, 
+			context=context, tracefile=tracefile)
+	    except service_error, e:
+		raise RPCException(desc=e.desc, code=e.code, 
+			errstr=e.code_string())
+    else:
+	raise RuntimeError("Unknown RPC transport: %s" % transport)
+
+    if responseBody:
+	if responseBody in resp: return resp[responseBody]
+	else: raise RuntimeError("No body in response??")
+    else:
+	return resp
+
+def get_experiment_names(eid):
+    """
+    eid is the experimentID entry in one of a dict representing a fedd message.
+    Pull out the fedid and localname from that hash and return them as fedid,
+    localid)
+    """
+    fedid = None
+    local = None
+    for id in eid or []:
+	for k in id.keys():
+	    if k =='fedid':
+		fedid = id[k]
+	    elif k =='localname':
+		local = id[k]
+
+    return (fedid, local)
+
+class info_format:
+    def __init__(self, out=sys.stdout):
+	self.out = out
+	self.key = {
+		'vis': 'vis', 
+		'vtopo': 'vtopo',
+		'federant': 'federant',
+		'experimentdescription': 'experimentdescription',
+		'id': 'experimentID',
+		'status': 'experimentStatus',
+		'log': 'allocationLog',
+		'embedding': 'embedding',
+	    }
+	self.formatter = {
+		'vis':  self.print_vis_or_vtopo('vis', self.out),
+		'vtopo':  self.print_vis_or_vtopo('vtopo', self.out),
+		'federant':  self.print_xml,
+		'experimentdescription':  self.print_xml,
+		'id': self.print_id,
+		'status': self.print_string,
+		'log': self.print_string,
+		'embedding':  self.print_xml,
+	    }
+
+    def print_string(self, d):
+	"""
+	Print the string to the class output.
+	"""
+	print >>self.out, d
+
+    def print_id(self, d):
+	"""
+	d is an array of ID dicts.  Print each one to the class output.
+	"""
+	for id in d or []:
+	    for k, i in id.items():
+		print >>self.out, "%s: %s" % (k, i)
+
+    def print_xml(self, d):
+	"""
+	Very simple ugly xml formatter of the kinds of dicts that come back
+	from services.
+	"""
+	if isinstance(d, dict):
+	    for k, v in d.items():
+		print >>self.out, "<%s>" % k
+		self.print_xml(v)
+		print >>self.out, "</%s>" % k
+	elif isinstance(d, list):
+	    for x in d:
+		self.print_xml(x)
+	else:
+	    print >>self.out, d
+
+
+    class print_vis_or_vtopo:
+	"""
+	Print the retrieved data is a simple xml representation of the dict.
+	"""
+	def __init__(self, top, out=sys.stdout):
+	    self.xml = top
+	    self.out = out
+
+	def __call__(self, d, out=sys.stdout):
+	    str = "<%s>\n" % self.xml
+	    for t in ('node', 'lan'):
+		if d.has_key(t): 
+		    for x in d[t]:
+			str += "<%s>" % t
+			for k in x.keys():
+			    str += "<%s>%s</%s>" % (k, x[k],k)
+			str += "</%s>\n" % t
+	    str+= "</%s>" % self.xml
+	    print >>self.out, str
+
+    def __call__(self, d, r):
+	"""
+	Print the data of type r (one of the keys of key) to the class output.
+	"""
+	k = self.key.get(r, None)
+	if k:
+	    if k in d: self.formatter[r](d[k])
+	    else: raise RuntimeError("Bad response: no %s" %k)
+	else:
+	    raise RuntimeError("Don't understand datatype %s" %r)
+
Index: fedd/Makefile/federation/config_parser.py
===================================================================
--- fedd/Makefile/federation/config_parser.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/config_parser.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,45 @@
+#!/usr/local/bin/python
+
+from ConfigParser import *
+
+class config_parser(SafeConfigParser):
+    """
+    A SafeConfig parser with a more forgiving get attribute
+    """
+
+    def safe_get(self, sect, opt, method, default=None):
+	"""
+	If the option is present, return it, otherwise the default.
+	"""
+	if self.has_option(sect, opt): return method(self, sect, opt)
+	else: return default
+
+    def get(self, sect, opt, default=None):
+	"""
+	This returns the requested option or a given default.
+
+	It's more like getattr than get.
+	"""
+
+	return self.safe_get(sect, opt, SafeConfigParser.get, default)
+
+    def getint(self, sect, opt, default=0):
+	"""
+	Returns the selected option as an int or a default.
+	"""
+
+	return self.safe_get(sect, opt, SafeConfigParser.getint, default)
+
+    def getfloat(self, sect, opt, default=0.0):
+	"""
+	Returns the selected option as an int or a default.
+	"""
+
+	return self.safe_get(sect, opt, SafeConfigParser.getfloat, default)
+
+    def getboolean(self, sect, opt, default=False):
+	"""
+	Returns the selected option as a boolean or a default.
+	"""
+
+	return self.safe_get(sect, opt, SafeConfigParser.getboolean, default)
Index: fedd/Makefile/federation/deter_impl.py
===================================================================
--- fedd/Makefile/federation/deter_impl.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/deter_impl.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,128 @@
+#!/usr/local/bin/python
+
+import sys
+
+import emulab_access
+import dragon_access
+import protogeni_access
+import deter_internal_access
+import skeleton_access
+
+from experiment_control import experiment_control_local
+from ns2topdl import ns2topdl_local
+from util import read_simple_accessdb
+from fedid import fedid
+
+from authorizer import authorizer, abac_authorizer
+
+class deter_impl:
+    """
+    The implementation of access control based on mapping users to projects.
+
+    Users can be mapped to existing projects or have projects created
+    dynamically.  This implements both direct requests and proxies.
+    """
+    # Used by the SOAP caller
+    soap_namespaces = ('http://www.isi.edu/fedd.wsdl',
+	    'http://www.isi.edu/fedd_internal.wsdl')
+
+    def __init__(self, config=None):
+	"""
+	Initializer.  Uses the parsed configuration to create appropriate
+	components.
+	"""
+	self.soap_services = { }
+	self.xmlrpc_services = { }
+	self.auth = None
+
+	if config:
+	    self.cert_file = config.get("globals", "cert_file");
+	    self.cert_pwd = config.get("globals", "cert_pwd");
+	    self.trusted_certs = config.get("globals", "trusted_certs");
+	    self.access_type = config.get("globals", "access_type", "emulab")
+	    self.auth_type = config.get("globals", "auth_type", "legacy")
+
+	    for mp in config.get("globals", "module_path","").split(":"):
+		sys.path.append(mp)
+
+	    if self.auth_type == 'legacy':
+		self.auth = authorizer()
+	    elif self.auth_type == 'abac':
+		auth_url = config.get('globals', 'auth_url')
+		if not auth_url:
+		    raise RuntimeError("auth_url required for ABAC " + \
+			    "authorization")
+		if self.cert_file:
+		    me = fedid(file=self.cert_file)
+		else:
+		    raise RuntimeError("ABAC authorization needs a " +\
+			    "certificate file")
+		self.auth= abac_authorizer(url=auth_url, 
+			cert_file=self.cert_file, cert_pwd=self.cert_pwd,
+			trusted_certs=self.trusted_certs, me=me)
+	    else:
+		raise RuntimeError("Unknown authorizer type %s" % \
+			self.auth_type)
+
+	    access_db = config.get("globals", "accessdb")
+
+	    if access_db:
+		try:
+		    read_simple_accessdb(access_db, self.auth)
+		except EnvironmentError, e:
+		    raise RuntimeError(
+			    "Error reading accessDB %s: %s" % (access_db, e))
+		except ValueError, e:
+		    raise RuntimeError("%s" % e)
+
+	    if config.has_section("access"):
+		if self.access_type == "emulab":
+		    self.access = emulab_access.access(config, self.auth)
+		elif self.access_type == "dragon":
+		    self.access = dragon_access.access(config, self.auth)
+		elif self.access_type == "protogeni":
+		    self.access = protogeni_access.access(config, self.auth)
+		elif self.access_type == "deter_internal":
+		    self.access = deter_internal_access.access(config,
+			    self.auth)
+		elif self.access_type == "skel":
+		    self.access = skeleton_access.access(config, self.auth)
+		else:
+		    try:
+			exec 'from %s import access as plugin_access' \
+				% self.access_type
+			self.access = plugin_access(config, self.auth)
+		    except ImportError, e:
+			raise RuntimeError(
+				"Unknown access_type: %s (import failed: %s)" \
+					% (self.access_type, e))
+		self.soap_services.update(self.access.soap_services) 
+		self.xmlrpc_services.update(self.access.xmlrpc_services) 
+	    else:
+		self.access = None
+
+	    if config.has_section("experiment_control"):
+		self.experiment = \
+			experiment_control_local(config, self.auth)
+		# Tell the experiment control where local access control is and
+		# what testbeds it pertains to.
+		if getattr(self, 'access', None):
+		    for t in self.access.testbed:
+			self.experiment.local_access[t] = self.access
+
+		self.soap_services.update(self.experiment.soap_services) 
+		self.xmlrpc_services.update(self.experiment.xmlrpc_services) 
+		self.get_handler = self.experiment.get_handler
+	    else:
+		if self.access and getattr(self.access, 'get_handler', None):
+		    self.get_handler = self.access.get_handler
+		else:
+		    self.get_handler = None
+
+	    if config.has_section("ns2topdl"):
+		self.ns2topdl = ns2topdl_local(config, self.auth)
+		self.soap_services.update(self.ns2topdl.soap_services) 
+		self.xmlrpc_services.update(self.ns2topdl.xmlrpc_services) 
+
+def new_feddservice(config):
+    return deter_impl(config)
Index: fedd/Makefile/federation/deter_internal_access.py
===================================================================
--- fedd/Makefile/federation/deter_internal_access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/deter_internal_access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,366 @@
+#!/usr/local/bin/python
+
+import os,sys
+import re
+import string
+import copy
+import pickle
+import logging
+import time
+
+from threading import Thread, Lock
+from subprocess import Popen, call, PIPE, STDOUT
+
+from util import *
+from allocate_project import allocate_project_local, allocate_project_remote
+from access_project import access_project
+from fedid import fedid, generate_fedid
+from authorizer import authorizer
+from service_error import service_error
+from remote_service import xmlrpc_handler, soap_handler, service_caller
+
+import httplib
+import tempfile
+from urlparse import urlparse
+
+import topdl
+import list_log
+import proxy_emulab_segment
+import local_emulab_segment
+
+from access import access_base
+
+# Make log messages disappear if noone configures a fedd logger
+class nullHandler(logging.Handler):
+    def emit(self, record): pass
+
+fl = logging.getLogger("fedd.access")
+fl.addHandler(nullHandler())
+
+class access(access_base):
+    @staticmethod
+    def parse_vlans(v, log=None):
+	"""
+	Parse a vlan parameter into a set of vlan ids.  Comma separated
+	sequences of vlan ranges are acceptable.
+	"""
+	# the vlans can be a single integer, a comma separated list or a
+	# comma separated lists of dashed ranges.  E.g 100 or 100,300 or
+	# 100,300-305,400
+	vset = set()
+	if v:
+	    for v in [ x.strip() for x in v.split(",") ]:
+		try:
+		    if v.count("-") == 1:
+			f, t = v.split("-", 1)
+			for i in range(int(f), int(t)+1):
+			    vset.add(i)
+		    else:
+			vset.add(int(v))
+		except ValueError:
+		    if log:
+			log.warn("Invalid expression in vlan list: %s" % v)
+	return vset
+
+    def __init__(self, config=None, auth=None):
+        """
+        Initializer.  Pulls parameters out of the ConfigParser's access section.
+        """
+
+	access_base.__init__(self, config, auth)
+	self.domain = config.get("access", "domain")
+	vlan_str = config.get("access", "vlans")
+	self.vlans = self.parse_vlans(vlan_str)
+
+        self.attrs = { }
+        self.access = { }
+	# State is a dict of dicts indexed by segment fedid that includes the
+	# owners of the segment as fedids (who can manipulate it, key: owners),
+	# the repo dir/user for the allocation (key: user),  Current allocation
+	# log (key: log), and GRI of the reservation once made (key: gri)
+        self.log = logging.getLogger("fedd.access")
+        set_log_level(config, "access", self.log)
+
+        if config.has_option("access", "accessdb"):
+            self.read_access(config.get("access", "accessdb"))
+
+	# Add the ownership attributes to the authorizer.  Note that the
+	# indices of the allocation dict are strings, but the attributes are
+	# fedids, so there is a conversion.
+	self.state_lock.acquire()
+	for k in self.state.keys():
+	    for o in self.state[k].get('owners', []):
+		self.auth.set_attribute(o, fedid(hexstr=k))
+	    self.auth.set_attribute(fedid(hexstr=k),fedid(hexstr=k))
+	    # If the allocation has a vlan assigned, remove it from the
+	    # available vlans
+	    v = self.state[k].get('vlan', None)
+	    if v:
+		self.vlans.discard(v)
+	self.state_lock.release()
+
+	self.lookup_access = self.lookup_access_base
+
+	self.call_GetValue= service_caller('GetValue')
+	self.call_SetValue= service_caller('SetValue')
+
+        self.soap_services = {\
+            'RequestAccess': soap_handler("RequestAccess", self.RequestAccess),
+            'ReleaseAccess': soap_handler("ReleaseAccess", self.ReleaseAccess),
+            'StartSegment': soap_handler("StartSegment", self.StartSegment),
+            'TerminateSegment': soap_handler("TerminateSegment", 
+		self.TerminateSegment),
+            }
+        self.xmlrpc_services =  {\
+            'RequestAccess': xmlrpc_handler('RequestAccess',
+                self.RequestAccess),
+            'ReleaseAccess': xmlrpc_handler('ReleaseAccess',
+                self.ReleaseAccess),
+            'StartSegment': xmlrpc_handler("StartSegment", self.StartSegment),
+            'TerminateSegment': xmlrpc_handler('TerminateSegment',
+                self.TerminateSegment),
+            }
+
+    def RequestAccess(self, req, fid):
+        """
+        Handle the access request.  Proxy if not for us.
+
+        Parse out the fields and make the allocations or rejections if for us,
+        otherwise, assuming we're willing to proxy, proxy the request out.
+        """
+
+        # The dance to get into the request body
+        if req.has_key('RequestAccessRequestBody'):
+            req = req['RequestAccessRequestBody']
+        else:
+            raise service_error(service_error.req, "No request!?")
+
+	found, match = self.lookup_access(req, fid)
+	# keep track of what's been added
+	allocID, alloc_cert = generate_fedid(subj="alloc", log=self.log)
+	aid = unicode(allocID)
+
+	self.state_lock.acquire()
+	self.state[aid] = { }
+	self.state[aid]['user'] = found
+	self.state[aid]['owners'] = [ fid ]
+	self.state[aid]['vlan'] = None
+	self.write_state()
+	self.state_lock.release()
+	self.auth.set_attribute(fid, allocID)
+	self.auth.set_attribute(allocID, allocID)
+
+	try:
+	    f = open("%s/%s.pem" % (self.certdir, aid), "w")
+	    print >>f, alloc_cert
+	    f.close()
+	except EnvironmentError, e:
+	    raise service_error(service_error.internal, 
+		    "Can't open %s/%s : %s" % (self.certdir, aid, e))
+	return { 'allocID': { 'fedid': allocID } }
+
+    def ReleaseAccess(self, req, fid):
+        # The dance to get into the request body
+        if req.has_key('ReleaseAccessRequestBody'):
+            req = req['ReleaseAccessRequestBody']
+        else:
+            raise service_error(service_error.req, "No request!?")
+
+	# Local request
+	try:
+	    if req['allocID'].has_key('localname'):
+		auth_attr = aid = req['allocID']['localname']
+	    elif req['allocID'].has_key('fedid'):
+		aid = unicode(req['allocID']['fedid'])
+		auth_attr = req['allocID']['fedid']
+	    else:
+		raise service_error(service_error.req,
+			"Only localnames and fedids are understood")
+	except KeyError:
+	    raise service_error(service_error.req, "Badly formed request")
+
+	self.log.debug("[access] deallocation requested for %s", aid)
+	if not self.auth.check_attribute(fid, auth_attr):
+	    self.log.debug("[access] deallocation denied for %s", aid)
+	    raise service_error(service_error.access, "Access Denied")
+
+	self.state_lock.acquire()
+	if self.state.has_key(aid):
+	    self.log.debug("Found allocation for %s" %aid)
+	    del self.state[aid]
+	    self.write_state()
+	    self.state_lock.release()
+	    # And remove the access cert
+	    cf = "%s/%s.pem" % (self.certdir, aid)
+	    self.log.debug("Removing %s" % cf)
+	    os.remove(cf)
+	    return { 'allocID': req['allocID'] } 
+	else:
+	    self.state_lock.release()
+	    raise service_error(service_error.req, "No such allocation")
+
+    def extract_parameters(self, top):
+	"""
+	DETER's internal networking currently supports a fixed capacity link
+	between two endpoints.  Those endpoints may specify a VPN (or list or
+	range) to use.  This extracts the and vpn preferences from the segments
+	(as attributes) and the capacity of the connection as given by the
+	substrate.  The two endpoints VLAN choices are intersected to get set
+	of VLANs that are acceptable (no VLAN requiremnets means any is
+	acceptable).
+	"""
+	segments = filter(lambda x: isinstance(x, topdl.Segment), top.elements)
+
+	if len(segments) != 2 or len(top.substrates) != 1:
+	    raise service_error(service_error.req,
+		    "Requests to DRAGON must have exactlty two segments " +\
+			    "and one substrate")
+
+	vlans = set()
+	for s in segments:
+	    v = s.get_attribute('vlans')
+	    vlans &= self.parse_vlans(v)
+
+	if len(vlans) == 0:
+	    vlans = None
+
+	sub = top.substrates[0]
+	if sub.capacity:
+	    cap = int(sub.capacity.rate / 1000.0)
+	    if cap < 1: cap = 1
+	else:
+	    cap = 100
+
+	return cap, vlans
+
+    def export_store_info(self, cf, vlan, connInfo):
+	"""
+	For the export requests in the connection info, install the peer names
+	at the experiment controller via SetValue calls.
+	"""
+
+	for c in connInfo:
+	    for p in [ p for p in c.get('parameter', []) \
+		    if p.get('type', '') == 'output']:
+
+		if p.get('name', '') != 'vlan_id':
+		    self.log.error("Unknown export parameter: %s" % \
+			    p.get('name'))
+		    continue
+
+		k = p.get('key', None)
+		surl = p.get('store', None)
+		if surl and k:
+		    value = "%s" % vlan
+		    req = { 'name': k, 'value': value }
+		    self.call_SetValue(surl, req, cf)
+		else:
+		    self.log.error("Bad export request: %s" % p)
+
+    def StartSegment(self, req, fid):
+	err = None  # Any service_error generated after tmpdir is created
+	rv = None   # Return value from segment creation
+
+	try:
+	    req = req['StartSegmentRequestBody']
+	    topref = req['segmentdescription']['topdldescription']
+	except KeyError:
+	    raise service_error(server_error.req, "Badly formed request")
+
+	auth_attr = req['allocID']['fedid']
+	aid = "%s" % auth_attr
+	attrs = req.get('fedAttr', [])
+	if not self.auth.check_attribute(fid, auth_attr):
+	    raise service_error(service_error.access, "Access denied")
+	else:
+	    # See if this is a replay of an earlier succeeded StartSegment -
+	    # sometimes SSL kills 'em.  If so, replay the response rather than
+	    # redoing the allocation.
+	    self.state_lock.acquire()
+	    retval = self.state[aid].get('started', None)
+	    self.state_lock.release()
+	    if retval:
+		self.log.warning("Duplicate StartSegment for %s: " % aid + \
+			"replaying response")
+		return retval
+
+	certfile = "%s/%s.pem" % (self.certdir, aid)
+
+	if topref: topo = topdl.Topology(**topref)
+	else:
+	    raise service_error(service_error.req, 
+		    "Request missing segmentdescription'")
+
+	connInfo = req.get('connection', [])
+
+	cap, vlans = self.extract_parameters(topo)
+
+	# No vlans passes in, consider any vlan acceptable
+	if not vlans: 
+	    vlans = self.vlans
+
+	avail = self.vlans | vlans
+
+	if len(avail) != 0:
+	    vlan_no = avail.pop()
+	    self.vlans.discard(vlan_no)
+	else:
+	    raise service_error(service_error.federant, "No vlan available")
+
+	self.export_store_info(certfile, vlan_no, connInfo)
+
+
+	# This annotation isn't strictly necessary, but may help in debugging
+	rtopo = topo.clone()
+	for s in rtopo.substrates:
+	    s.set_attribute('vlan', vlan_no)
+
+	# Grab the log (this is some anal locking, but better safe than
+	# sorry)
+	self.state_lock.acquire()
+	self.state[aid]['vlan'] = vlan_no
+	logv = "Allocated vlan: %d" % vlan_no
+	# It's possible that the StartSegment call gets retried (!). 
+	# if the 'started' key is in the allocation, we'll return it rather
+	# than redo the setup.
+	self.state[aid]['started'] = { 
+		'allocID': req['allocID'],
+		'allocationLog': logv,
+		'segmentdescription': { 'topdldescription': rtopo.to_dict() }
+		}
+	retval = copy.deepcopy(self.state[aid]['started'])
+	self.write_state()
+	self.state_lock.release()
+
+	return retval
+
+    def TerminateSegment(self, req, fid):
+	try:
+	    req = req['TerminateSegmentRequestBody']
+	except KeyError:
+	    raise service_error(server_error.req, "Badly formed request")
+
+	auth_attr = req['allocID']['fedid']
+	aid = "%s" % auth_attr
+
+	self.log.debug("Terminate request for %s" %aid)
+	if not self.auth.check_attribute(fid, auth_attr):
+	    raise service_error(service_error.access, "Access denied")
+
+	self.state_lock.acquire()
+	if self.state.has_key(aid):
+	    vlan_no = self.state[aid].get('vlan', None)
+	else:
+	    vlan_no = None
+	self.state_lock.release()
+	self.log.debug("Stop segment for vlan: %s" % vlan_no)
+
+	if not vlan_no:
+	    raise service_error(service_error.internal, 
+		    "Can't find assigfned vlan for for %s" % aid)
+   
+	self.vlans.add(vlan_no)
+	self.state_lock.acquire()
+	self.state[aid]['vlan'] = None
+	self.state_lock.release()
+	return { 'allocID': req['allocID'] }
Index: fedd/Makefile/federation/dragon_access.py
===================================================================
--- fedd/Makefile/federation/dragon_access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/dragon_access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,615 @@
+#!/usr/local/bin/python
+
+import os,sys
+import re
+import string
+import copy
+import pickle
+import logging
+import time
+
+from threading import Thread, Lock
+from subprocess import Popen, call, PIPE, STDOUT
+from access import access_base
+
+from util import *
+from allocate_project import allocate_project_local, allocate_project_remote
+from access_project import access_project
+from fedid import fedid, generate_fedid
+from authorizer import authorizer
+from service_error import service_error
+from remote_service import xmlrpc_handler, soap_handler, service_caller
+
+import httplib
+import tempfile
+from urlparse import urlparse
+
+import topdl
+import list_log
+import proxy_emulab_segment
+import local_emulab_segment
+
+
+# Make log messages disappear if noone configures a fedd logger
+class nullHandler(logging.Handler):
+    def emit(self, record): pass
+
+fl = logging.getLogger("fedd.access")
+fl.addHandler(nullHandler())
+
+class access(access_base):
+    """
+    The implementation of access control based on mapping users to projects.
+
+    Users can be mapped to existing projects or have projects created
+    dynamically.  This implements both direct requests and proxies.
+    """
+
+    def __init__(self, config=None, auth=None):
+        """
+        Initializer.  Pulls parameters out of the ConfigParser's access section.
+        """
+	access_base.__init__(self, config, auth)
+
+	self.cli_dir = config.get("access", "cli_dir")
+	self.axis2_home = config.get("access", "axis2_home")
+	self.idc_url = config.get("access", "idc")
+	self.domain = config.get("access", "domain")
+	self.duration = config.getint("access", "duration", 120)
+
+        self.access = { }
+	if not (self.cli_dir and self.axis2_home and self.idc_url):
+	    self.log.error("Must specify all of cli_dir, axis2_home, " +\
+		    "idc in the [access] section of the configuration")
+
+        if config.has_option("access", "accessdb"):
+            self.read_access(config.get("access", "accessdb"), self.make_repo)
+
+	# Add the ownership attributes to the authorizer.  Note that the
+	# indices of the allocation dict are strings, but the attributes are
+	# fedids, so there is a conversion.
+	self.state_lock.acquire()
+	for k in self.state.keys():
+	    for o in self.state[k].get('owners', []):
+		self.auth.set_attribute(o, fedid(hexstr=k))
+	    self.auth.set_attribute(fedid(hexstr=k),fedid(hexstr=k))
+	self.state_lock.release()
+
+	self.lookup_access = self.lookup_access_base
+
+	self.call_GetValue= service_caller('GetValue')
+	self.call_SetValue= service_caller('SetValue')
+
+        self.soap_services = {\
+            'RequestAccess': soap_handler("RequestAccess", self.RequestAccess),
+            'ReleaseAccess': soap_handler("ReleaseAccess", self.ReleaseAccess),
+            'StartSegment': soap_handler("StartSegment", self.StartSegment),
+            'TerminateSegment': soap_handler("TerminateSegment", 
+		self.TerminateSegment),
+            }
+        self.xmlrpc_services =  {\
+            'RequestAccess': xmlrpc_handler('RequestAccess',
+                self.RequestAccess),
+            'ReleaseAccess': xmlrpc_handler('ReleaseAccess',
+                self.ReleaseAccess),
+            'StartSegment': xmlrpc_handler("StartSegment", self.StartSegment),
+            'TerminateSegment': xmlrpc_handler('TerminateSegment',
+                self.TerminateSegment),
+            }
+
+    @staticmethod
+    def make_repo(s):
+	"""
+	Get the repo directory from an access line.  This is removing the ()
+	from the string.
+	"""
+	rv = s.strip()
+	if rv.startswith('(') and rv.endswith(')'): return rv[1:-1]
+	else: raise self.parse_error("Repo should be in parens");
+
+    def RequestAccess(self, req, fid):
+        """
+        Handle the access request.
+
+        Parse out the fields and make the allocations or rejections if for us,
+        otherwise, assuming we're willing to proxy, proxy the request out.
+        """
+
+        # The dance to get into the request body
+        if req.has_key('RequestAccessRequestBody'):
+            req = req['RequestAccessRequestBody']
+        else:
+            raise service_error(service_error.req, "No request!?")
+
+        if req.has_key('destinationTestbed'):
+            dt = unpack_id(req['destinationTestbed'])
+
+	# Request for this fedd
+	found, match = self.lookup_access(req, fid)
+	# keep track of what's been added
+	allocID, alloc_cert = generate_fedid(subj="alloc", log=self.log)
+	aid = unicode(allocID)
+
+	self.state_lock.acquire()
+	self.state[aid] = { }
+	self.state[aid]['user'] = found
+	self.state[aid]['owners'] = [ fid ]
+	self.write_state()
+	self.state_lock.release()
+	self.auth.set_attribute(fid, allocID)
+	self.auth.set_attribute(allocID, allocID)
+
+	try:
+	    f = open("%s/%s.pem" % (self.certdir, aid), "w")
+	    print >>f, alloc_cert
+	    f.close()
+	except EnvironmentError, e:
+	    raise service_error(service_error.internal, 
+		    "Can't open %s/%s : %s" % (self.certdir, aid, e))
+	return { 'allocID': { 'fedid': allocID } }
+
+    def ReleaseAccess(self, req, fid):
+        # The dance to get into the request body
+        if req.has_key('ReleaseAccessRequestBody'):
+            req = req['ReleaseAccessRequestBody']
+        else:
+            raise service_error(service_error.req, "No request!?")
+
+	try:
+	    if req['allocID'].has_key('localname'):
+		auth_attr = aid = req['allocID']['localname']
+	    elif req['allocID'].has_key('fedid'):
+		aid = unicode(req['allocID']['fedid'])
+		auth_attr = req['allocID']['fedid']
+	    else:
+		raise service_error(service_error.req,
+			"Only localnames and fedids are understood")
+	except KeyError:
+	    raise service_error(service_error.req, "Badly formed request")
+
+	self.log.debug("[access] deallocation requested for %s", aid)
+	if not self.auth.check_attribute(fid, auth_attr):
+	    self.log.debug("[access] deallocation denied for %s", aid)
+	    raise service_error(service_error.access, "Access Denied")
+
+	self.state_lock.acquire()
+	if self.state.has_key(aid):
+	    self.log.debug("Found allocation for %s" %aid)
+	    del self.state[aid]
+	    self.write_state()
+	    self.state_lock.release()
+	    # And remove the access cert
+	    cf = "%s/%s.pem" % (self.certdir, aid)
+	    self.log.debug("Removing %s" % cf)
+	    os.remove(cf)
+	    return { 'allocID': req['allocID'] } 
+	else:
+	    self.state_lock.release()
+	    raise service_error(service_error.req, "No such allocation")
+
+    def extract_parameters(self, top):
+	"""
+	DRAGON currently supports a fixed capacity link between two endpoints.
+	Those endpoints may specify a VPN (or list or range) to use.  This
+	extracts the DRAGON endpoints and vpn preferences from the segments (as
+	attributes) and the capacity of the connection as given by the
+	substrate.  The two endpoints VLAN choices are intersected to get set
+	of VLANs that are acceptable (no VLAN requiremnets means any is
+	acceptable).
+	"""
+	segments = filter(lambda x: isinstance(x, topdl.Segment), top.elements)
+
+	if len(segments) != 2 or len(top.substrates) != 1:
+	    raise service_error(service_error.req,
+		    "Requests to DRAGON must have exactlty two segments " +\
+			    "and one substrate")
+
+	ends = [ ]
+	for s in segments:
+	    ep = s.get_attribute('dragon_endpoint')
+	    if not ep:
+		raise service_error(service_error.req, 
+			"Missing DRAGON endpoint for %s" % s.id)
+	    v = s.get_attribute('vlans')
+	    vlans = None
+	    vset = set()
+	    # the vlans can be a single integer, a comma separated list or a
+	    # comma separated lists of dashed ranges.  E.g 100 or 100,300 or
+	    # 100,300-305,400
+	    if v:
+		if v.count(",") > 0 :
+		    vl = [ x.strip() for x in v.split(",") ]
+		else:
+		    vl = [ v ]
+		for v in vl:
+		    try:
+			if v.count("-")> 0:
+			    f, t = v.split("-", 1)
+			    for i in range(int(f), int(t)+1):
+				vset.add(i)
+			else:
+			    vset.add(int(v))
+		    except ValueError:
+			raise service_error(service_error.req, 
+				"VLAN tags must be integers (%s)" %s.name)
+	    if len(vset) > 0:
+		if vlans: vlans &= vest
+		else: vlans = vset
+	    ends.append(ep)
+
+
+	sub = top.substrates[0]
+	if sub.capacity:
+	    cap = int(sub.capacity.rate / 1000.0)
+	    if cap < 1: cap = 1
+	else:
+	    cap = 100
+
+
+	# DRAGON's command line tool barfs if the source (ends[0]) is not in
+	# the domain controlled by the IDC.  This code ensures this situation.
+	if self.domain and not ends[0].endswith(self.domain):
+	    hold = ends[0]
+	    for i, e in enumerate(ends):
+		if i == 0: continue
+		if e.endswith(self.domain):
+		    ends[0] = e
+		    ends[i] = hold
+		    break
+	    else:
+		raise service_error(service_error.req, 
+			"No endpoint in my domain")
+
+
+	return cap, ends[0], ends[1], vlans
+
+    def oscars_create_vpn(self, repo, fr, to, cap, v, start, end, log):
+
+	gri_re = re.compile("GRI:\s*(.*)", re.IGNORECASE)
+	status_re = re.compile("Status:\s*(.*)", re.IGNORECASE)
+
+	cmd = ['env', 'AXIS2_HOME=%s' % self.axis2_home, './run.sh', 
+	    'createReservation', '-repo',  repo , '-url', self.idc_url,
+	    '-l2source', fr, '-l2dest', to, '-bwidth', "%s" % cap,
+	    '-vlan', "%s" % v, '-desc', 'fedd created connection',
+	    '-pathsetup', 'timer-automatic', '-start', "%d" % int(start),
+	    '-end', "%d" % int(end)]
+	log.debug("[start_segment]: %s" % " ".join(cmd))
+	if not self.create_debug: 
+	    p = Popen(cmd, cwd=self.cli_dir, stdout=PIPE, stderr=STDOUT,
+		    close_fds=True)
+	    for line in p.stdout:
+		m = status_re.match(line)
+		if m:
+		    status = m.group(1)
+		    continue
+		m = gri_re.match(line)
+		if m:
+		    gri = m.group(1)
+		    continue
+	    rv = p.wait()
+	else: 
+	    rv = 0
+	    status = 'ACCEPTED'
+	    gri = 'debug_gri'
+
+	return (rv, status, gri)
+
+    def oscars_query_vpn(self, repo, gri, v, log):
+	"""
+	Call the oscars query command from the command line and parse out the
+	data to see if the current request succeeded.  This is a lot of fiddly
+	code to do  a pretty simple thing.
+	"""
+	status_re = re.compile("Status:\s*(.*)", re.IGNORECASE)
+	source_re = re.compile("Source\s+Endpoint:\s*(.*)", re.IGNORECASE)
+	dest_re = re.compile("Destination\s+Endpoint:\s*(.*)", re.IGNORECASE)
+	path_re = re.compile("Path:")
+
+	cmd = ['env', 'AXIS2_HOME=%s' % self.axis2_home, './run.sh', 
+	    'query', '-repo',  repo , '-url', self.idc_url, '-gri', gri]
+	log.debug("[start_segment]: %s" % " ".join(cmd))
+	if not self.create_debug:
+	    # Really do the query
+	    p = Popen(cmd, cwd=self.cli_dir, stdout=PIPE, stderr=STDOUT,
+		    close_fds=True)
+	    in_path = False
+	    vpn1 = None
+	    vpn2 = None
+	    src = None
+	    dest = None
+	    for line in p.stdout:
+		if not in_path:
+		    m = status_re.match(line)
+		    if m: 
+			status = m.group(1)
+			continue
+		    m = source_re.match(line)
+		    if m:
+			src = m.group(1)
+			continue
+		    m = dest_re.match(line)
+		    if m:
+			dest = m.group(1)
+			continue
+		    m = path_re.match(line)
+		    if m:
+			in_path = True
+			if src and dest:
+			    vpn1_re = re.compile(
+				    "\s*%s,\s*\w+\s*,\s*(\d+)" % \
+					    src.replace("*", "\*"))
+			    vpn2_re = re.compile(
+				    "\s*%s,\s*\w+\s*,\s*(\d+)" % \
+					    dest.replace("*", "\*"))
+			else:
+			    raise service_error(service_error.internal, 
+				    "Strange output from query")
+		else:
+		    m = vpn1_re.match(line)
+		    if m:
+			vpn1 = m.group(1)
+			continue
+		    m = vpn2_re.match(line)
+		    if m:
+			vpn2 = m.group(1)
+			continue
+	    rv = p.wait()
+	    # Make sure that OSCARS did what we expected.
+	    if vpn1 == vpn2:
+		if v is not None:
+		    if int(vpn1) == v:
+			vlan_no = int(v)
+		    else:
+			raise service_error(service_error.federant, 
+				"Unexpected vlan assignment")
+		else:
+		    vlan_no = int(v or 0)
+	    else:
+		raise service_error(service_error.internal,
+			"Different VPNs on DRAGON ends")
+	    log.debug("Status: %s" % status or "none")
+	else:
+	    rv = 0
+	    status = 'ACTIVE'
+	    vlan_no = int(v or 1)
+
+	return (rv, status, vlan_no)
+
+
+
+    def start_segment(self, repo, fr, to, cap, vpns=None, start=None, end=None,
+	    log=None):
+	"""
+	Do the actual work of creating the dragon connecton.
+	"""
+	waiting_states = ('ACCEPTED', 'INSETUP', 'INCREATE', 'PENDING')
+	if not log: log = self.log
+
+	if not vpns:
+	    vpns = [ None, None, None, None, None]
+
+	if not start:
+	    start = time.time()
+	if not end:
+	    end = start + self.duration *60
+
+
+	status = None
+	gri = None
+	rv = None
+	vlan_no = None
+	for v in vpns:
+	    rv, status, gri = self.oscars_create_vpn(repo, fr, to, cap, v, 
+		    start, end, log)
+	    # Reservation in progress.  Poll the IDC until we know the outcome
+	    while status in waiting_states:
+		rv, status, vlan_no = self.oscars_query_vpn(repo, gri, v, log)
+		if status in waiting_states:
+		    time.sleep(45)
+	    if status in ('ACTIVE', 'FINISHED', 'CANCELLED'):
+		break
+
+	if (rv == 0 and gri and vlan_no and status == 'ACTIVE'):
+	    self.log.debug("made reservation %s %s" % (gri, vlan_no))
+	    return gri, vlan_no
+	else:
+	    raise service_error(service_error.federant, 
+		    "Cannot make reservation")
+
+    def stop_segment(self, repo, gri, log=None):
+	"""
+	Terminate the reservation.
+	"""
+	if not log: log = self.log
+	cmd = ['env', 'AXIS2_HOME=%s' % self.axis2_home, './run.sh', 
+	    'cancel', '-repo',  repo , '-url', self.idc_url, '-gri', gri]
+
+
+	self.log.debug("[stop_segment]: %s" % " ".join(cmd))
+	if not self.create_debug:
+	    try:
+		f = open("/dev/null", "w")
+		call(cmd, cwd=self.cli_dir, stdout=f, stderr=f, close_fds=True)
+		f.close()
+	    except EnvironmentError, e:
+		raise service_error(service_error.internal, 
+			"Failed to open /dev/null: %s" % e)
+
+    def export_store_info(self, cf, vlan, connInfo):
+	"""
+	For the export requests in the connection info, install the peer names
+	at the experiment controller via SetValue calls.
+	"""
+
+	for c in connInfo:
+	    for p in [ p for p in c.get('parameter', []) \
+		    if p.get('type', '') == 'output']:
+
+		if p.get('name', '') != 'vlan_id':
+		    self.log.error("Unknown export parameter: %s" % \
+			    p.get('name'))
+		    continue
+
+		k = p.get('key', None)
+		surl = p.get('store', None)
+		if surl and k:
+		    value = "%s" % vlan
+		    req = { 'name': k, 'value': value }
+		    print "calling SetValue %s %s" % (surl, req)
+		    self.call_SetValue(surl, req, cf)
+		else:
+		    self.log.error("Bad export request: %s" % p)
+
+    def initialize_experiment_info(self, aid, ename):
+	repo = None
+	self.state_lock.acquire()
+	if aid in self.state:
+	    repo = self.state[aid].get('user', None)
+	    self.state[aid]['log'] = [ ]
+	    # Create a logger that logs to the experiment's state object as
+	    # well as to the main log file.
+	    alloc_log = logging.getLogger('fedd.access.%s' % ename)
+	    h = logging.StreamHandler(
+		    list_log.list_log(self.state[aid]['log']))
+	    # XXX: there should be a global one of these rather than
+	    # repeating the code.
+	    h.setFormatter(logging.Formatter(
+		"%(asctime)s %(name)s %(message)s", '%d %b %y %H:%M:%S'))
+	    alloc_log.addHandler(h)
+	    self.write_state()
+	self.state_lock.release()
+	return (repo, alloc_log)
+
+    def finalize_experiment(self, topo, vlan_no, gri, aid, alloc_id):
+	"""
+	Place the relevant information in the global state block, and prepare
+	the response.
+	"""
+	rtopo = topo.clone()
+	for s in rtopo.substrates:
+	    s.set_attribute('vlan', vlan_no)
+	    s.set_attribute('gri', gri)
+
+	# Grab the log (this is some anal locking, but better safe than
+	# sorry)
+	self.state_lock.acquire()
+	self.state[aid]['gri'] = gri
+	logv = "".join(self.state[aid]['log'])
+	# It's possible that the StartSegment call gets retried (!). 
+	# if the 'started' key is in the allocation, we'll return it rather
+	# than redo the setup.
+	self.state[aid]['started'] = { 
+		'allocID': alloc_id,
+		'allocationLog': logv,
+		'segmentdescription': {
+		    'topdldescription': rtopo.to_dict()
+		    },
+		}
+	retval = copy.deepcopy(self.state[aid]['started'])
+	self.write_state()
+	self.state_lock.release()
+
+	return retval
+
+    def StartSegment(self, req, fid):
+	err = None  # Any service_error generated after tmpdir is created
+	rv = None   # Return value from segment creation
+
+	try:
+	    req = req['StartSegmentRequestBody']
+	    topref = req['segmentdescription']['topdldescription']
+	except KeyError:
+	    raise service_error(server_error.req, "Badly formed request")
+
+	auth_attr = req['allocID']['fedid']
+	aid = "%s" % auth_attr
+	attrs = req.get('fedAttr', [])
+	if not self.auth.check_attribute(fid, auth_attr):
+	    raise service_error(service_error.access, "Access denied")
+	else:
+	    # See if this is a replay of an earlier succeeded StartSegment -
+	    # sometimes SSL kills 'em.  If so, replay the response rather than
+	    # redoing the allocation.
+	    self.state_lock.acquire()
+	    retval = self.state[aid].get('started', None)
+	    self.state_lock.release()
+	    if retval:
+		self.log.warning("Duplicate StartSegment for %s: " % aid + \
+			"replaying response")
+		return retval
+
+	certfile = "%s/%s.pem" % (self.certdir, aid)
+
+	if topref:
+	    topo = topdl.Topology(**topref)
+	else:
+	    raise service_error(service_error.req, 
+		    "Request missing segmentdescription'")
+
+	connInfo = req.get('connection', [])
+
+	cap, src, dest, vlans = self.extract_parameters(topo)
+
+	for a in attrs:
+	    if a['attribute'] == 'experiment_name':
+		ename = a['value']
+		break
+	else: ename = aid
+	
+	repo, alloc_log = self.initialize_experiment_info(aid, ename)
+
+	if not repo:
+	    raise service_error(service_error.internal, 
+		    "Can't find creation user for %s" % aid)
+
+	gri, vlan_no = self.start_segment(repo, src, dest, cap, vlans,
+		log=alloc_log)
+
+	self.export_store_info(certfile, vlan_no, connInfo)
+
+
+	if gri:
+	    return self.finalize_experiment(topo, vlan_no, gri, aid, 
+		    req['allocID'])
+	elif err:
+	    raise service_error(service_error.federant,
+		    "Swapin failed: %s" % err)
+	else:
+	    raise service_error(service_error.federant, "Swapin failed")
+
+    def TerminateSegment(self, req, fid):
+	try:
+	    req = req['TerminateSegmentRequestBody']
+	except KeyError:
+	    raise service_error(server_error.req, "Badly formed request")
+
+	auth_attr = req['allocID']['fedid']
+	aid = "%s" % auth_attr
+
+	self.log.debug("Terminate request for %s" %aid)
+	attrs = req.get('fedAttr', [])
+	if not self.auth.check_attribute(fid, auth_attr):
+	    raise service_error(service_error.access, "Access denied")
+
+	self.state_lock.acquire()
+	if self.state.has_key(aid):
+	    gri = self.state[aid].get('gri', None)
+	    user = self.state[aid].get('user', None)
+	else:
+	    gri = None
+	    user = None
+	self.state_lock.release()
+	self.log.debug("Stop segment for user: %s gre %s" %(user, gri))
+
+	if not gri:
+	    raise service_error(service_error.internal, 
+		    "Can't find gri for %s" % aid)
+
+	if not user:
+	    raise service_error(service_error.internal, 
+		    "Can't find creation user for %s" % aid)
+    
+	self.log.debug("Stop segment for GRI: %s" %gri)
+	self.stop_segment(user, gri)
+	return { 'allocID': req['allocID'] }
Index: fedd/Makefile/federation/emulab_access.py
===================================================================
--- fedd/Makefile/federation/emulab_access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/emulab_access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,1102 @@
+#!/usr/local/bin/python
+
+import os,sys
+import stat # for chmod constants
+import re
+import random
+import string
+import copy
+import pickle
+import logging
+import subprocess
+import traceback
+
+from threading import *
+from M2Crypto.SSL import SSLError
+
+from access import access_base
+
+from util import *
+from allocate_project import allocate_project_local, allocate_project_remote
+from access_project import access_project
+from fedid import fedid, generate_fedid
+from authorizer import authorizer
+from service_error import service_error
+from remote_service import xmlrpc_handler, soap_handler, service_caller
+
+import httplib
+import tempfile
+from urlparse import urlparse
+
+import topdl
+import list_log
+import proxy_emulab_segment
+import local_emulab_segment
+
+
+# Make log messages disappear if noone configures a fedd logger
+class nullHandler(logging.Handler):
+    def emit(self, record): pass
+
+fl = logging.getLogger("fedd.access")
+fl.addHandler(nullHandler())
+
+class access(access_base):
+    """
+    The implementation of access control based on mapping users to projects.
+
+    Users can be mapped to existing projects or have projects created
+    dynamically.  This implements both direct requests and proxies.
+    """
+
+    def __init__(self, config=None, auth=None):
+        """
+        Initializer.  Pulls parameters out of the ConfigParser's access section.
+        """
+
+	access_base.__init__(self, config, auth)
+
+        self.allow_proxy = config.getboolean("access", "allow_proxy")
+
+        self.domain = config.get("access", "domain")
+	self.userconfdir = config.get("access","userconfdir")
+	self.userconfcmd = config.get("access","userconfcmd")
+	self.userconfurl = config.get("access","userconfurl")
+	self.federation_software = config.get("access", "federation_software")
+	self.portal_software = config.get("access", "portal_software")
+	self.local_seer_software = config.get("access", "local_seer_software")
+	self.local_seer_image = config.get("access", "local_seer_image")
+	self.local_seer_start = config.get("access", "local_seer_start")
+	self.seer_master_start = config.get("access", "seer_master_start")
+	self.ssh_privkey_file = config.get("access","ssh_privkey_file")
+	self.ssh_pubkey_file = config.get("access","ssh_pubkey_file")
+	self.ssh_port = config.get("access","ssh_port") or "22"
+	self.boss = config.get("access", "boss")
+	self.xmlrpc_cert = config.get("access", "xmlrpc_cert")
+	self.xmlrpc_certpw = config.get("access", "xmlrpc_certpw")
+
+	self.dragon_endpoint = config.get("access", "dragon")
+	self.dragon_vlans = config.get("access", "dragon_vlans")
+	self.deter_internal = config.get("access", "deter_internal")
+
+	self.tunnel_config = config.getboolean("access", "tunnel_config")
+	self.portal_command = config.get("access", "portal_command")
+	self.portal_image = config.get("access", "portal_image")
+	self.portal_type = config.get("access", "portal_type") or "pc"
+	self.portal_startcommand = config.get("access", "portal_startcommand")
+	self.node_startcommand = config.get("access", "node_startcommand")
+
+	self.federation_software = self.software_list(self.federation_software)
+	self.portal_software = self.software_list(self.portal_software)
+	self.local_seer_software = self.software_list(self.local_seer_software)
+
+	self.access_type = self.access_type.lower()
+	if self.access_type == 'remote_emulab':
+	    self.start_segment = proxy_emulab_segment.start_segment
+	    self.stop_segment = proxy_emulab_segment.stop_segment
+	elif self.access_type == 'local_emulab':
+	    self.start_segment = local_emulab_segment.start_segment
+	    self.stop_segment = local_emulab_segment.stop_segment
+	else:
+	    self.start_segment = None
+	    self.stop_segment = None
+
+        self.restricted = [ ]
+        self.access = { }
+        if config.has_option("access", "accessdb"):
+            self.read_access(config.get("access", "accessdb"))
+        tb = config.get('access', 'testbed')
+        if tb: self.testbed = [ t.strip() for t in tb.split(',') ]
+        else: self.testbed = [ ]
+
+        if config.has_option("access", "accessdb"):
+            self.read_access(config.get("access", "accessdb"), 
+		    self.make_access_project)
+
+	# read_state in the base_class
+	self.state_lock.acquire()
+	for a  in ('allocation', 'projects', 'keys', 'types'):
+	    if a not in self.state:
+		self.state[a] = { }
+	self.allocation = self.state['allocation']
+	self.projects = self.state['projects']
+	self.keys = self.state['keys']
+	self.types = self.state['types']
+	# Add the ownership attributes to the authorizer.  Note that the
+	# indices of the allocation dict are strings, but the attributes are
+	# fedids, so there is a conversion.
+	for k in self.allocation.keys():
+	    for o in self.allocation[k].get('owners', []):
+		self.auth.set_attribute(o, fedid(hexstr=k))
+	    if self.allocation[k].has_key('userconfig'):
+		sfid = self.allocation[k]['userconfig']
+		fid = fedid(hexstr=sfid)
+		self.auth.set_attribute(fid, "/%s" % sfid)
+	self.state_lock.release()
+	self.exports = {
+		'SMB': self.export_SMB,
+		'seer': self.export_seer,
+		'tmcd': self.export_tmcd,
+		'userconfig': self.export_userconfig,
+		'project_export': self.export_project_export,
+		'local_seer_control': self.export_local_seer,
+		'seer_master': self.export_seer_master,
+		'hide_hosts': self.export_hide_hosts,
+		}
+
+	if not self.local_seer_image or not self.local_seer_software or \
+		not self.local_seer_start:
+	    if 'local_seer_control' in self.exports:
+		del self.exports['local_seer_control']
+
+	if not self.local_seer_image or not self.local_seer_software or \
+		not self.seer_master_start:
+	    if 'seer_master' in self.exports:
+		del self.exports['seer_master']
+
+
+        self.soap_services = {\
+            'RequestAccess': soap_handler("RequestAccess", self.RequestAccess),
+            'ReleaseAccess': soap_handler("ReleaseAccess", self.ReleaseAccess),
+            'StartSegment': soap_handler("StartSegment", self.StartSegment),
+            'TerminateSegment': soap_handler("TerminateSegment",
+		self.TerminateSegment),
+            }
+        self.xmlrpc_services =  {\
+            'RequestAccess': xmlrpc_handler('RequestAccess',
+                self.RequestAccess),
+            'ReleaseAccess': xmlrpc_handler('ReleaseAccess',
+                self.ReleaseAccess),
+            'StartSegment': xmlrpc_handler("StartSegment", self.StartSegment),
+            'TerminateSegment': xmlrpc_handler('TerminateSegment',
+                self.TerminateSegment),
+            }
+
+	self.call_SetValue = service_caller('SetValue')
+	self.call_GetValue = service_caller('GetValue', log=self.log)
+
+        if not config.has_option("allocate", "uri"):
+            self.allocate_project = \
+                allocate_project_local(config, auth)
+        else:
+            self.allocate_project = \
+                allocate_project_remote(config, auth)
+
+
+        # If the project allocator exports services, put them in this object's
+        # maps so that classes that instantiate this can call the services.
+        self.soap_services.update(self.allocate_project.soap_services)
+        self.xmlrpc_services.update(self.allocate_project.xmlrpc_services)
+
+    @staticmethod
+    def make_access_project(str):
+	"""
+	Convert a string of the form (id[:resources:resouces], id, id) into an
+	access_project.  This is called by read_access to convert to local
+	attributes.  It returns a tuple of the form (project, user, user) where
+	users may be names or fedids.
+	"""
+        def parse_name(n):
+            if n.startswith('fedid:'): return fedid(hexstr=n[len('fedid:'):])
+            else: return n
+
+	str = str.strip()
+	if str.startswith('(') and str.endswith(')'):
+	    str = str[1:-1]
+	    names = [ s.strip() for s in str.split(",")]
+	    if len(names) > 3:
+		raise self.parse_error("More than three fields in name")
+	    first = names[0].split(":")
+	    if first == 'fedid:':
+		del first[0]
+		first[0] = fedid(hexstr=first[0])
+	    names[0] = access_project(first[0], first[1:])
+
+	    for i in range(1,2):
+		names[i] = parse_name(names[i])
+
+	    return tuple(names)
+	else:
+	    raise self.parse_error('Bad mapping (unbalanced parens)')
+
+
+    # RequestAccess support routines
+
+    def lookup_access(self, req, fid):
+        """
+	Look up the local access control information mapped to this fedid and
+	credentials.  In this case it is a (project, create_user, access_user)
+	triple, and a triple of three booleans indicating which, if any will
+	need to be dynamically created.  Finally a list of owners for that
+	allocation is returned.
+
+	lookup_access_base pulls the first triple out, and it is parsed by this
+	routine into the boolean map.  Owners is always the controlling fedid.
+        """
+        # Return values
+        rp = access_project(None, ())
+        ru = None
+        # This maps a valid user to the Emulab projects and users to use
+        found, match = self.lookup_access_base(req, fid)
+	tb, project, user = match
+        
+        if found == None:
+            raise service_error(service_error.access,
+                    "Access denied - cannot map access")
+
+        # resolve <dynamic> and <same> in found
+        dyn_proj = False
+        dyn_create_user = False
+        dyn_service_user = False
+
+        if found[0].name == "<same>":
+            if project != None:
+                rp.name = project
+            else : 
+                raise service_error(\
+                        service_error.server_config,
+                        "Project matched <same> when no project given")
+        elif found[0].name == "<dynamic>":
+            rp.name = None
+            dyn_proj = True
+        else:
+            rp.name = found[0].name
+        rp.node_types = found[0].node_types;
+
+        if found[1] == "<same>":
+            if user_match == "<any>":
+                if user != None: rcu = user[0]
+                else: raise service_error(\
+                        service_error.server_config,
+                        "Matched <same> on anonymous request")
+            else:
+                rcu = user_match
+        elif found[1] == "<dynamic>":
+            rcu = None
+            dyn_create_user = True
+        else:
+            rcu = found[1]
+        
+        if found[2] == "<same>":
+            if user_match == "<any>":
+                if user != None: rsu = user[0]
+                else: raise service_error(\
+                        service_error.server_config,
+                        "Matched <same> on anonymous request")
+            else:
+                rsu = user_match
+        elif found[2] == "<dynamic>":
+            rsu = None
+            dyn_service_user = True
+        else:
+            rsu = found[2]
+
+        return (rp, rcu, rsu), (dyn_create_user, dyn_service_user, dyn_proj),\
+                [ fid ]
+
+    def do_project_allocation(self, dyn, project, user):
+	"""
+	Call the project allocation routines and return the info.
+	"""
+	if dyn: 
+	    # Compose the dynamic project request 
+	    # (only dynamic, dynamic currently allowed)
+	    preq = { 'AllocateProjectRequestBody': \
+			{ 'project' : {\
+			    'user': [ \
+			    { \
+				'access': [ { 
+				    'sshPubkey': self.ssh_pubkey_file } ],
+				 'role': "serviceAccess",\
+			    }, \
+			    { \
+				'access': [ { 
+				    'sshPubkey': self.ssh_pubkey_file } ],
+				 'role': "experimentCreation",\
+			    }, \
+			    ], \
+			    }\
+			}\
+		    }
+	    return self.allocate_project.dynamic_project(preq)
+	else:
+	    preq = {'StaticProjectRequestBody' : \
+		    { 'project': \
+			{ 'name' : { 'localname' : project },\
+			  'user' : [ \
+			    {\
+				'userID': { 'localname' : user }, \
+				'access': [ { 
+				    'sshPubkey': self.ssh_pubkey_file } ],
+				'role': 'experimentCreation'\
+			    },\
+			    {\
+				'userID': { 'localname' : user}, \
+				'access': [ { 
+				    'sshPubkey': self.ssh_pubkey_file } ],
+				'role': 'serviceAccess'\
+			    },\
+			]}\
+		    }\
+	    }
+	    return self.allocate_project.static_project(preq)
+
+    def save_project_state(self, aid, ap, dyn, owners):
+	"""
+	Parse out and save the information relevant to the project created for
+	this experiment.  That info is largely in ap and owners.  dyn indicates
+	that the project was created dynamically.  Return the user and project
+	names.
+	"""
+	self.state_lock.acquire()
+	self.allocation[aid] = { }
+	try:
+	    pname = ap['project']['name']['localname']
+	except KeyError:
+	    pname = None
+
+	if dyn:
+	    if not pname:
+		self.state_lock.release()
+		raise service_error(service_error.internal,
+			"Misformed allocation response?")
+	    if pname in self.projects: self.projects[pname] += 1
+	    else: self.projects[pname] = 1
+	    self.allocation[aid]['project'] = pname
+	else:
+	    # sproject is a static project associated with this allocation.
+	    self.allocation[aid]['sproject'] = pname
+
+	self.allocation[aid]['keys'] = [ ]
+
+	try:
+	    for u in ap['project']['user']:
+		uname = u['userID']['localname']
+		if u['role'] == 'experimentCreation':
+		    self.allocation[aid]['user'] = uname
+		for k in [ k['sshPubkey'] for k in u['access'] \
+			if k.has_key('sshPubkey') ]:
+		    kv = "%s:%s" % (uname, k)
+		    if self.keys.has_key(kv): self.keys[kv] += 1
+		    else: self.keys[kv] = 1
+		    self.allocation[aid]['keys'].append((uname, k))
+	except KeyError:
+	    self.state_lock.release()
+	    raise service_error(service_error.internal,
+		    "Misformed allocation response?")
+
+	self.allocation[aid]['owners'] = owners
+	self.write_state()
+	self.state_lock.release()
+	return (pname, uname)
+
+    # End of RequestAccess support routines
+
+    def RequestAccess(self, req, fid):
+        """
+        Handle the access request.  Proxy if not for us.
+
+        Parse out the fields and make the allocations or rejections if for us,
+        otherwise, assuming we're willing to proxy, proxy the request out.
+        """
+
+        def gateway_hardware(h):
+            if h == 'GWTYPE': return self.portal_type or 'GWTYPE'
+            else: return h
+
+	def get_export_project(svcs):
+	    """
+	    if the service requests includes one to export a project, return
+	    that project.
+	    """
+	    rv = None
+	    for s in svcs:
+		if s.get('name', '') == 'project_export' and \
+			s.get('visibility', '') == 'export':
+		    if not rv: 
+			for a in s.get('feddAttr', []):
+			    if a.get('attribute', '') == 'project' \
+				    and 'value' in a:
+				rv = a['value']
+		    else:
+			raise service_error(service_error, access, 
+				'Requesting multiple project exports is ' + \
+					'not supported');
+	    return rv
+
+        # The dance to get into the request body
+        if req.has_key('RequestAccessRequestBody'):
+            req = req['RequestAccessRequestBody']
+        else:
+            raise service_error(service_error.req, "No request!?")
+
+
+	found, dyn, owners = self.lookup_access(req, fid)
+	ap = None
+
+	# if this includes a project export request and the exported
+	# project is not the access project, access denied.
+	if 'service' in req:
+	    ep = get_export_project(req['service'])
+	    if ep and ep != found[0].name:
+		raise service_error(service_error.access,
+			"Cannot export %s" % ep)
+
+	if self.ssh_pubkey_file:
+	    ap = self.do_project_allocation(dyn[1], found[0].name, found[1])
+	else:
+	    raise service_error(service_error.internal, 
+		    "SSH access parameters required")
+	# keep track of what's been added
+	allocID, alloc_cert = generate_fedid(subj="alloc", log=self.log)
+	aid = unicode(allocID)
+
+	pname, uname = self.save_project_state(aid, ap, dyn[1], owners)
+
+	services, svc_state = self.export_services(req.get('service',[]),
+		pname, uname)
+	self.state_lock.acquire()
+	# Store services state in global state
+	for k, v in svc_state.items():
+	    self.allocation[aid][k] = v
+	self.write_state()
+	self.state_lock.release()
+	# Give the owners the right to change this allocation
+	for o in owners:
+	    self.auth.set_attribute(o, allocID)
+	try:
+	    f = open("%s/%s.pem" % (self.certdir, aid), "w")
+	    print >>f, alloc_cert
+	    f.close()
+	except EnvironmentError, e:
+	    raise service_error(service_error.internal, 
+		    "Can't open %s/%s : %s" % (self.certdir, aid, e))
+	resp = self.build_access_response({ 'fedid': allocID } ,
+		ap, services)
+	return resp
+
+    def do_release_project(self, del_project, del_users, del_types):
+	"""
+	If a project and users has to be deleted, make the call.
+	"""
+	msg = { 'project': { }}
+	if del_project:
+	    msg['project']['name']= {'localname': del_project}
+	users = [ ]
+	for u in del_users.keys():
+	    users.append({ 'userID': { 'localname': u },\
+		'access' :  \
+			[ {'sshPubkey' : s } for s in del_users[u]]\
+	    })
+	if users: 
+	    msg['project']['user'] = users
+	if len(del_types) > 0:
+	    msg['resources'] = { 'node': \
+		    [ {'hardware': [ h ] } for h in del_types ]\
+		}
+	if self.allocate_project.release_project:
+	    msg = { 'ReleaseProjectRequestBody' : msg}
+	    self.allocate_project.release_project(msg)
+
+    def ReleaseAccess(self, req, fid):
+        # The dance to get into the request body
+        if req.has_key('ReleaseAccessRequestBody'):
+            req = req['ReleaseAccessRequestBody']
+        else:
+            raise service_error(service_error.req, "No request!?")
+
+	try:
+	    if req['allocID'].has_key('localname'):
+		auth_attr = aid = req['allocID']['localname']
+	    elif req['allocID'].has_key('fedid'):
+		aid = unicode(req['allocID']['fedid'])
+		auth_attr = req['allocID']['fedid']
+	    else:
+		raise service_error(service_error.req,
+			"Only localnames and fedids are understood")
+	except KeyError:
+	    raise service_error(service_error.req, "Badly formed request")
+
+	self.log.debug("[access] deallocation requested for %s", aid)
+	if not self.auth.check_attribute(fid, auth_attr):
+	    self.log.debug("[access] deallocation denied for %s", aid)
+	    raise service_error(service_error.access, "Access Denied")
+
+	# If we know this allocation, reduce the reference counts and
+	# remove the local allocations.  Otherwise report an error.  If
+	# there is an allocation to delete, del_users will be a dictonary
+	# of sets where the key is the user that owns the keys in the set.
+	# We use a set to avoid duplicates.  del_project is just the name
+	# of any dynamic project to delete.  We're somewhat lazy about
+	# deleting authorization attributes.  Having access to something
+	# that doesn't exist isn't harmful.
+	del_users = { }
+	del_project = None
+	del_types = set()
+
+	self.state_lock.acquire()
+	if aid in self.allocation:
+	    self.log.debug("Found allocation for %s" %aid)
+	    for k in self.allocation[aid]['keys']:
+		kk = "%s:%s" % k
+		self.keys[kk] -= 1
+		if self.keys[kk] == 0:
+		    if not del_users.has_key(k[0]):
+			del_users[k[0]] = set()
+		    del_users[k[0]].add(k[1])
+		    del self.keys[kk]
+
+	    if 'project' in self.allocation[aid]:
+		pname = self.allocation[aid]['project']
+		self.projects[pname] -= 1
+		if self.projects[pname] == 0:
+		    del_project = pname
+		    del self.projects[pname]
+
+	    if 'types' in self.allocation[aid]:
+		for t in self.allocation[aid]['types']:
+		    self.types[t] -= 1
+		    if self.types[t] == 0:
+			if not del_project: del_project = t[0]
+			del_types.add(t[1])
+			del self.types[t]
+
+	    del self.allocation[aid]
+	    self.write_state()
+	    self.state_lock.release()
+	    # If we actually have resources to deallocate, prepare the call.
+	    if del_project or del_users:
+		self.do_release_project(del_project, del_users, del_types)
+	    # And remove the access cert
+	    cf = "%s/%s.pem" % (self.certdir, aid)
+	    self.log.debug("Removing %s" % cf)
+	    os.remove(cf)
+	    return { 'allocID': req['allocID'] } 
+	else:
+	    self.state_lock.release()
+	    raise service_error(service_error.req, "No such allocation")
+
+    # These are subroutines for StartSegment
+    def generate_ns2(self, topo, expfn, softdir, connInfo):
+	"""
+	Convert topo into an ns2 file, decorated with appropriate commands for
+	the particular testbed setup.  Convert all requests for software, etc
+	to point at the staged copies on this testbed and add the federation
+	startcommands.
+	"""
+	class dragon_commands:
+	    """
+	    Functor to spit out approrpiate dragon commands for nodes listed in
+	    the connectivity description.  The constructor makes a dict mapping
+	    dragon nodes to their parameters and the __call__ checks each
+	    element in turn for membership.
+	    """
+	    def __init__(self, map):
+		self.node_info = map
+
+	    def __call__(self, e):
+		s = ""
+		if isinstance(e, topdl.Computer):
+		    if self.node_info.has_key(e.name):
+			info = self.node_info[e.name]
+			for ifname, vlan, type in info:
+			    for i in e.interface:
+				if i.name == ifname:
+				    addr = i.get_attribute('ip4_address')
+				    subs = i.substrate[0]
+				    break
+			    else:
+				raise service_error(service_error.internal,
+					"No interface %s on element %s" % \
+						(ifname, e.name))
+			    # XXX: do netmask right
+			    if type =='link':
+				s = ("tb-allow-external ${%s} " + \
+					"dragonportal ip %s vlan %s " + \
+					"netmask 255.255.255.0\n") % \
+					(topdl.to_tcl_name(e.name), addr, vlan)
+			    elif type =='lan':
+				s = ("tb-allow-external ${%s} " + \
+					"dragonportal " + \
+					"ip %s vlan %s usurp %s\n") % \
+					(topdl.to_tcl_name(e.name), addr, 
+						vlan, subs)
+			    else:
+				raise service_error(service_error_internal,
+					"Unknown DRAGON type %s" % type)
+		return s
+
+	class not_dragon:
+	    """
+	    Return true if a node is in the given map of dragon nodes.
+	    """
+	    def __init__(self, map):
+		self.nodes = set(map.keys())
+
+	    def __call__(self, e):
+		return e.name not in self.nodes
+
+	# Main line of generate_ns2
+	t = topo.clone()
+
+	# Create the map of nodes that need direct connections (dragon
+	# connections) from the connInfo
+	dragon_map = { }
+	for i in [ i for i in connInfo if i['type'] == 'transit']:
+	    for a in i.get('fedAttr', []):
+		if a['attribute'] == 'vlan_id':
+		    vlan = a['value']
+		    break
+	    else:
+		raise service_error(service_error.internal, "No vlan tag")
+	    members = i.get('member', [])
+	    if len(members) > 1: type = 'lan'
+	    else: type = 'link'
+
+	    try:
+		for m in members:
+		    if m['element'] in dragon_map:
+			dragon_map[m['element']].append(( m['interface'], 
+			    vlan, type))
+		    else:
+			dragon_map[m['element']] = [( m['interface'], 
+			    vlan, type),]
+	    except KeyError:
+		raise service_error(service_error.req,
+			"Missing connectivity info")
+
+	# Weed out the things we aren't going to instantiate: Segments, portal
+	# substrates, and portal interfaces.  (The copy in the for loop allows
+	# us to delete from e.elements in side the for loop).  While we're
+	# touching all the elements, we also adjust paths from the original
+	# testbed to local testbed paths and put the federation commands into
+	# the start commands
+	for e in [e for e in t.elements]:
+	    if isinstance(e, topdl.Segment):
+		t.elements.remove(e)
+	    if isinstance(e, topdl.Computer):
+		self.add_kit(e, self.federation_software)
+		if e.get_attribute('portal') and self.portal_startcommand:
+		    # Add local portal support software
+		    self.add_kit(e, self.portal_software)
+		    # Portals never have a user-specified start command
+		    e.set_attribute('startup', self.portal_startcommand)
+		elif self.node_startcommand:
+		    if e.get_attribute('startup'):
+			e.set_attribute('startup', "%s \\$USER '%s'" % \
+				(self.node_startcommand, 
+				    e.get_attribute('startup')))
+		    else:
+			e.set_attribute('startup', self.node_startcommand)
+
+		dinf = [i[0] for i in dragon_map.get(e.name, []) ]
+		# Remove portal interfaces that do not connect to DRAGON
+		e.interface = [i for i in e.interface \
+			if not i.get_attribute('portal') or i.name in dinf ]
+	    # Fix software paths
+	    for s in getattr(e, 'software', []):
+		s.location = re.sub("^.*/", softdir, s.location)
+
+	t.substrates = [ s.clone() for s in t.substrates ]
+	t.incorporate_elements()
+
+	# Customize the ns2 output for local portal commands and images
+	filters = []
+
+	if self.dragon_endpoint:
+	    add_filter = not_dragon(dragon_map)
+	    filters.append(dragon_commands(dragon_map))
+	else:
+	    add_filter = None
+
+	if self.portal_command:
+	    filters.append(topdl.generate_portal_command_filter(
+		self.portal_command, add_filter=add_filter))
+
+	if self.portal_image:
+	    filters.append(topdl.generate_portal_image_filter(
+		self.portal_image))
+
+	if self.portal_type:
+	    filters.append(topdl.generate_portal_hardware_filter(
+		self.portal_type))
+
+	# Convert to ns and write it out
+	expfile = topdl.topology_to_ns2(t, filters)
+	try:
+	    f = open(expfn, "w")
+	    print >>f, expfile
+	    f.close()
+	except EnvironmentError:
+	    raise service_error(service_error.internal,
+		    "Cannot write experiment file %s: %s" % (expfn,e))
+
+    def export_store_info(self, cf, proj, ename, connInfo):
+	"""
+	For the export requests in the connection info, install the peer names
+	at the experiment controller via SetValue calls.
+	"""
+
+	for c in connInfo:
+	    for p in [ p for p in c.get('parameter', []) \
+		    if p.get('type', '') == 'output']:
+
+		if p.get('name', '') == 'peer':
+		    k = p.get('key', None)
+		    surl = p.get('store', None)
+		    if surl and k and k.index('/') != -1:
+			value = "%s.%s.%s%s" % \
+				(k[k.index('/')+1:], ename, proj, self.domain)
+			req = { 'name': k, 'value': value }
+			self.log.debug("Setting %s to %s on %s" % \
+				(k, value, surl))
+			self.call_SetValue(surl, req, cf)
+		    else:
+			self.log.error("Bad export request: %s" % p)
+		elif p.get('name', '') == 'ssh_port':
+		    k = p.get('key', None)
+		    surl = p.get('store', None)
+		    if surl and k:
+			req = { 'name': k, 'value': self.ssh_port }
+			self.log.debug("Setting %s to %s on %s" % \
+				(k, self.ssh_port, surl))
+			self.call_SetValue(surl, req, cf)
+		    else:
+			self.log.error("Bad export request: %s" % p)
+		else:
+		    self.log.error("Unknown export parameter: %s" % \
+			    p.get('name'))
+		    continue
+
+    def add_seer_node(self, topo, name, startup):
+	"""
+	Add a seer node to the given topology, with the startup command passed
+	in.  Used by configure seer_services.
+	"""
+	c_node = topdl.Computer(
+		name=name, 
+		os= topdl.OperatingSystem(
+		    attribute=[
+		    { 'attribute': 'osid', 
+			'value': self.local_seer_image },
+		    ]),
+		attribute=[
+		    { 'attribute': 'startup', 'value': startup },
+		    ]
+		)
+	self.add_kit(c_node, self.local_seer_software)
+	topo.elements.append(c_node)
+
+    def configure_seer_services(self, services, topo, softdir):
+	"""
+	Make changes to the topology required for the seer requests being made.
+	Specifically, add any control or master nodes required and set up the
+	start commands on the nodes to interconnect them.
+	"""
+	local_seer = False	# True if we need to add a control node
+	collect_seer = False	# True if there is a seer-master node
+	seer_master= False	# True if we need to add the seer-master
+	for s in services:
+	    s_name = s.get('name', '')
+	    s_vis = s.get('visibility','')
+
+	    if s_name  == 'local_seer_control' and s_vis == 'export':
+		local_seer = True
+	    elif s_name == 'seer_master':
+		if s_vis == 'import':
+		    collect_seer = True
+		elif s_vis == 'export':
+		    seer_master = True
+	
+	# We've got the whole picture now, so add nodes if needed and configure
+	# them to interconnect properly.
+	if local_seer or seer_master:
+	    # Copy local seer control node software to the tempdir
+	    for l, f in self.local_seer_software:
+		base = os.path.basename(f)
+		copy_file(f, "%s/%s" % (softdir, base))
+	# If we're collecting seers somewhere the controllers need to talk to
+	# the master.  In testbeds that export the service, that will be a
+	# local node that we'll add below.  Elsewhere it will be the control
+	# portal that will port forward to the exporting master.
+	if local_seer:
+	    if collect_seer:
+		startup = "%s -C %s" % (self.local_seer_start, "seer-master")
+	    else:
+		startup = self.local_seer_start
+	    self.add_seer_node(topo, 'control', startup)
+	# If this is the seer master, add that node, too.
+	if seer_master:
+	    self.add_seer_node(topo, 'seer-master', 
+		    "%s -R -n -R seer-master -R -A -R sink" % \
+			    self.seer_master_start)
+
+    def retrieve_software(self, topo, certfile, softdir):
+	"""
+	Collect the software that nodes in the topology need loaded and stage
+	it locally.  This implies retrieving it from the experiment_controller
+	and placing it into softdir.  Certfile is used to prove that this node
+	has access to that data (it's the allocation/segment fedid).  Finally
+	local portal and federation software is also copied to the same staging
+	directory for simplicity - all software needed for experiment creation
+	is in softdir.
+	"""
+	sw = set()
+	for e in topo.elements:
+	    for s in getattr(e, 'software', []):
+		sw.add(s.location)
+	for s in sw:
+	    self.log.debug("Retrieving %s" % s)
+	    try:
+		get_url(s, certfile, softdir)
+	    except:
+		t, v, st = sys.exc_info()
+		raise service_error(service_error.internal,
+			"Error retrieving %s: %s" % (s, v))
+
+	# Copy local federation and portal node software to the tempdir
+	for s in (self.federation_software, self.portal_software):
+	    for l, f in s:
+		base = os.path.basename(f)
+		copy_file(f, "%s/%s" % (softdir, base))
+
+
+    def initialize_experiment_info(self, attrs, aid, certfile, tmpdir):
+	"""
+	Gather common configuration files, retrieve or create an experiment
+	name and project name, and return the ssh_key filenames.  Create an
+	allocation log bound to the state log variable as well. 
+	"""
+	configs = set(('hosts', 'ssh_pubkey', 'ssh_secretkey'))
+	ename = None
+	pubkey_base = None
+	secretkey_base = None
+	proj = None
+	user = None
+	alloc_log = None
+
+	for a in attrs:
+	    if a['attribute'] in configs:
+		try:
+		    self.log.debug("Retrieving %s from %s" % \
+			    (a['attribute'], a['value']))
+		    get_url(a['value'], certfile, tmpdir)
+		except:
+		    t, v, st = sys.exc_info()
+		    raise service_error(service_error.internal,
+			    "Error retrieving %s: %s" % (a.get('value', ""), v))
+	    if a['attribute'] == 'ssh_pubkey':
+		pubkey_base = a['value'].rpartition('/')[2]
+	    if a['attribute'] == 'ssh_secretkey':
+		secretkey_base = a['value'].rpartition('/')[2]
+	    if a['attribute'] == 'experiment_name':
+		ename = a['value']
+
+	if not ename:
+	    ename = ""
+	    for i in range(0,5):
+		ename += random.choice(string.ascii_letters)
+	    self.log.warn("No experiment name: picked one randomly: %s" \
+		    % ename)
+
+	if not pubkey_base:
+	    raise service_error(service_error.req, 
+		    "No public key attribute")
+
+	if not secretkey_base:
+	    raise service_error(service_error.req, 
+		    "No secret key attribute")
+
+	self.state_lock.acquire()
+	if aid in self.allocation:
+	    proj = self.allocation[aid].get('project', None)
+	    if not proj: 
+		proj = self.allocation[aid].get('sproject', None)
+	    user = self.allocation[aid].get('user', None)
+	    self.allocation[aid]['experiment'] = ename
+	    self.allocation[aid]['log'] = [ ]
+	    # Create a logger that logs to the experiment's state object as
+	    # well as to the main log file.
+	    alloc_log = logging.getLogger('fedd.access.%s' % ename)
+	    h = logging.StreamHandler(
+		    list_log.list_log(self.allocation[aid]['log']))
+	    # XXX: there should be a global one of these rather than
+	    # repeating the code.
+	    h.setFormatter(logging.Formatter(
+		"%(asctime)s %(name)s %(message)s",
+			'%d %b %y %H:%M:%S'))
+	    alloc_log.addHandler(h)
+	    self.write_state()
+	self.state_lock.release()
+
+	if not proj:
+	    raise service_error(service_error.internal, 
+		    "Can't find project for %s" %aid)
+
+	if not user:
+	    raise service_error(service_error.internal, 
+		    "Can't find creation user for %s" %aid)
+
+	return (ename, proj, user, pubkey_base, secretkey_base, alloc_log)
+
+    def finalize_experiment(self, starter, topo, aid, alloc_id):
+	"""
+	Store key bits of experiment state in the global repository, including
+	the response that may need to be replayed, and return the response.
+	"""
+	# Copy the assigned names into the return topology
+	embedding = [ ]
+	for n in starter.node:
+	    embedding.append({ 
+		'toponame': n,
+		'physname': ["%s%s" %  (starter.node[n], self.domain)],
+		})
+	# Grab the log (this is some anal locking, but better safe than
+	# sorry)
+	self.state_lock.acquire()
+	logv = "".join(self.allocation[aid]['log'])
+	# It's possible that the StartSegment call gets retried (!). 
+	# if the 'started' key is in the allocation, we'll return it rather
+	# than redo the setup.
+	self.allocation[aid]['started'] = { 
+		'allocID': alloc_id,
+		'allocationLog': logv,
+		'segmentdescription': { 
+		    'topdldescription': topo.clone().to_dict()
+		    },
+		'embedding': embedding
+		}
+	retval = copy.copy(self.allocation[aid]['started'])
+	self.write_state()
+	self.state_lock.release()
+	return retval
+    
+    # End of StartSegment support routines
+
+    def StartSegment(self, req, fid):
+	err = None  # Any service_error generated after tmpdir is created
+	rv = None   # Return value from segment creation
+
+	try:
+	    req = req['StartSegmentRequestBody']
+	    auth_attr = req['allocID']['fedid']
+	    topref = req['segmentdescription']['topdldescription']
+	except KeyError:
+	    raise service_error(server_error.req, "Badly formed request")
+
+	connInfo = req.get('connection', [])
+	services = req.get('service', [])
+	aid = "%s" % auth_attr
+	attrs = req.get('fedAttr', [])
+	if not self.auth.check_attribute(fid, auth_attr):
+	    raise service_error(service_error.access, "Access denied")
+	else:
+	    # See if this is a replay of an earlier succeeded StartSegment -
+	    # sometimes SSL kills 'em.  If so, replay the response rather than
+	    # redoing the allocation.
+	    self.state_lock.acquire()
+	    retval = self.allocation[aid].get('started', None)
+	    self.state_lock.release()
+	    if retval:
+		self.log.warning("Duplicate StartSegment for %s: " % aid + \
+			"replaying response")
+		return retval
+
+	# A new request.  Do it.
+
+	if topref: topo = topdl.Topology(**topref)
+	else:
+	    raise service_error(service_error.req, 
+		    "Request missing segmentdescription'")
+	
+	certfile = "%s/%s.pem" % (self.certdir, auth_attr)
+        try:
+            tmpdir = tempfile.mkdtemp(prefix="access-")
+	    softdir = "%s/software" % tmpdir
+	    os.mkdir(softdir)
+        except EnvironmentError:
+            raise service_error(service_error.internal, "Cannot create tmp dir")
+
+	# Try block alllows us to clean up temporary files.
+	try:
+	    self.retrieve_software(topo, certfile, softdir)
+	    ename, proj, user, pubkey_base, secretkey_base, alloc_log = \
+		    self.initialize_experiment_info(attrs, aid, 
+			    certfile, tmpdir)
+
+	    # Set up userconf and seer if needed
+	    self.configure_userconf(services, tmpdir)
+	    self.configure_seer_services(services, topo, softdir)
+	    # Get and send synch store variables
+	    self.export_store_info(certfile, proj, ename, connInfo)
+	    self.import_store_info(certfile, connInfo)
+
+	    expfile = "%s/experiment.tcl" % tmpdir
+
+	    self.generate_portal_configs(topo, pubkey_base, 
+		    secretkey_base, tmpdir, proj, ename, connInfo, services)
+	    self.generate_ns2(topo, expfile, 
+		    "/proj/%s/software/%s/" % (proj, ename), connInfo)
+
+	    starter = self.start_segment(keyfile=self.ssh_privkey_file, 
+		    debug=self.create_debug, log=alloc_log, boss=self.boss,
+		    cert=self.xmlrpc_cert)
+	    rv = starter(self, ename, proj, user, expfile, tmpdir)
+	except service_error, e:
+	    err = e
+	except:
+	    t, v, st = sys.exc_info()
+	    err = service_error(service_error.internal, "%s: %s" % \
+		    (v, traceback.extract_tb(st)))
+
+        # Walk up tmpdir, deleting as we go
+	if self.cleanup: self.remove_dirs(tmpdir)
+	else: self.log.debug("[StartSegment]: not removing %s" % tmpdir)
+
+	if rv:
+	    return self.finalize_experiment(starter, topo, aid, req['allocID'])
+	elif err:
+	    raise service_error(service_error.federant,
+		    "Swapin failed: %s" % err)
+	else:
+	    raise service_error(service_error.federant, "Swapin failed")
+
+    def TerminateSegment(self, req, fid):
+	try:
+	    req = req['TerminateSegmentRequestBody']
+	except KeyError:
+	    raise service_error(server_error.req, "Badly formed request")
+
+	auth_attr = req['allocID']['fedid']
+	aid = "%s" % auth_attr
+	attrs = req.get('fedAttr', [])
+	if not self.auth.check_attribute(fid, auth_attr):
+	    raise service_error(service_error.access, "Access denied")
+
+	self.state_lock.acquire()
+	if aid in self.allocation:
+	    proj = self.allocation[aid].get('project', None)
+	    if not proj: 
+		proj = self.allocation[aid].get('sproject', None)
+	    user = self.allocation[aid].get('user', None)
+	    ename = self.allocation[aid].get('experiment', None)
+	else:
+	    proj = None
+	    user = None
+	    ename = None
+	self.state_lock.release()
+
+	if not proj:
+	    raise service_error(service_error.internal, 
+		    "Can't find project for %s" % aid)
+
+	if not user:
+	    raise service_error(service_error.internal, 
+		    "Can't find creation user for %s" % aid)
+	if not ename:
+	    raise service_error(service_error.internal, 
+		    "Can't find experiment name for %s" % aid)
+	stopper = self.stop_segment(keyfile=self.ssh_privkey_file,
+		debug=self.create_debug, boss=self.boss, cert=self.xmlrpc_cert)
+	stopper(self, user, proj, ename)
+	return { 'allocID': req['allocID'] }
Index: fedd/Makefile/federation/emulab_segment.py
===================================================================
--- fedd/Makefile/federation/emulab_segment.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/emulab_segment.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,210 @@
+#!/usr/local/bin/python
+
+import logging 
+import util
+
+from M2Crypto import SSL
+from M2Crypto.m2xmlrpclib import SSL_Transport
+from xmlrpclib import ServerProxy, dumps, loads, Fault, Error, Binary
+from M2Crypto.SSL import SSLError
+from M2Crypto.BIO import BIOError
+from socket import error as socket_error
+from socket import sslerror
+import httplib
+from federation.util import fedd_ssl_context
+from federation import service_error
+
+class emulab_segment:
+    def __init__(self, boss, cert):
+	self.ctxt = fedd_ssl_context(my_cert=cert)
+	self.ctxt.set_verify(SSL.verify_none, 10)
+	self.boss = boss
+	self.null = """
+set ns [new Simulator]
+source tb_compat.tcl
+
+set a [$ns node]
+
+$ns rtproto Session
+$ns run 
+"""
+	self.log =  getattr(self, 'log', None) 
+	self.debug = getattr(self, 'debug', False)
+
+    def emulab_call(self, method, params):
+	VERSION = 0.1
+	try:
+	    transport = SSL_Transport(self.ctxt)
+	    port = ServerProxy(self.boss, transport=transport)
+	    remote_method = getattr(port, method, None)
+	    if remote_method is not None:
+		resp = remote_method(VERSION, params)
+	    else:
+		raise service_error(service_error.internal, 
+			"Bad method: %s" % method)
+	except socket_error, e:
+	    raise service_error(service_error.connect, 
+		    "Cannot connect to %s: %s" % (url, e[1]))
+	    raise e
+	except BIOError, e:
+	    if self.log:
+		self.log.warn("BIO error contacting %s: %s" % (url, e))
+	    raise e
+	except sslerror, e:
+	    if self.log:
+		self.log.warn("SSL (socket) error contacting %s: %s" % 
+			(url, e))
+	    raise e
+	except SSLError, e:
+	    if self.log:
+		self.log.warn("SSL error contacting %s: %s" % (url, e))
+	    raise e
+	except httplib.HTTPException, e:
+	    if self.log:
+		self.log.warn("HTTP error contacting %s: %s" % (url, e))
+	    raise e
+	except Fault, f:
+	    raise service_error(f.faultCode, f.faultString)
+	except Error, e:
+	    raise service_error(service_error.protocol, 
+		    "Remote XMLRPC Fault: %s" % e)
+
+	code = resp.get('code', -1)
+	if code ==0:
+	    return (code, resp.get('value', None))
+	else:
+	    return (code, resp.get('output', None))
+
+    def get_state(self, pid, eid):
+	"""
+	Return the state of the experiment as reported by emulab
+	"""
+
+	if self.debug:
+	    state = 'swapped'
+	else:
+	    params =  { 'proj': pid, 'exp': eid }
+	    code, state = self.emulab_call('experiment.state', params)
+	    if code != 0:
+		state = 'none'
+
+	if self.log:
+	    self.log.debug("State is %s" % state)
+	return state
+
+    def make_null_experiment(self, pid, eid, tmpdir):
+	"""
+	Create a null copy of the experiment so that we capture any logs there
+	if the modify fails.  Emulab software discards the logs from a failed
+	startexp.
+	"""
+
+	if self.debug:
+	    if self.log:
+		self.log.debug("[make_null_experiment]: (debug) Creating experiment")
+	    return True
+	else:
+	    params = {
+		    'proj': pid,
+		    'exp': eid, 
+		    'nsfilestr': self.null,
+		    'batch': False,
+		    'idleswap': 0,
+		    'noidleswap_reason': 'Federated experiment',
+		    'noswapin': True,
+		    'wait': True
+		    }
+	    if self.log:
+		self.log.info("[make_null_experiment]: Creating experiment")
+	    code, value = self.emulab_call('experiment.startexp', params)
+
+	    if self.log:
+		if code == 0: 
+		    self.log.info('[make_null_experiment]: Create succeeded')
+		else: 
+		    self.log.error('[make_null_experiment]: Create failed: %s' \
+			    % value)
+
+	    return code == 0
+
+    def swap_exp(self, pid, eid, direction='out', wait=True):
+	"""
+	Swap experiment in.
+	"""
+	if self.debug:
+	    if self.log:
+		self.log.info("[swap_exp]: (debug) Swapping %s %s" % \
+			(eid, direction))
+	    return True
+	else:
+	    if self.log:
+		self.log.info("[swap_exp]: Swapping %s %s" % (eid, direction))
+	    params = {
+		    'proj': pid,
+		    'exp': eid,
+		    'direction': direction,
+		    'wait': wait,
+		    }
+	    code, value = self.emulab_call('experiment.swapexp', params)
+
+	    if self.log:
+		if code == 0: self.log.info('[swap_exp]: Swap succeeded')
+		else: self.log.error('[swap_exp]: Swap failed: %s' % value)
+
+	return code == 0
+
+    def modify_exp(self, pid, eid, tcl, wait=True):
+	if self.debug:
+	    self.log.info("[modify_exp]: (debug) Modifying %s" % eid)
+	    return True
+	else:
+	    self.log.info("[modify_exp]: Modifying %s" % eid)
+	    params = {
+		    'proj': pid,
+		    'exp': eid,
+		    'nsfilestr': tcl,
+		    'wait': wait,
+		    'reboot': True,
+		    'restart_eventsys': True,
+		    }
+	    code, value = self.emulab_call('experiment.modify', params)
+	    if self.log:
+		if code == 0: 
+		    self.log.info('[modify_exp]: Modify succeeded')
+		else: 
+		    self.log.error('[modify_exp]: Modify failed: %s' \
+			    % value)
+	    return code == 0
+
+
+    def get_mapping(self, pid, eid):
+	"""
+	Get the physical to virtual mapping from the expinfo command and save
+	it in the self.map member.
+	"""
+
+	if self.debug:
+	    if self.log:
+		self.log.info("[get_mapping] (debug) Generating mapping")
+		return True
+	else:
+	    if self.log:
+		self.log.info("[get_mapping] Generating mapping")
+
+	    params = {
+		    'proj': pid,
+		    'exp': eid,
+		    'aspect': 'mapping'
+		    }
+	    code, nodes = self.emulab_call('experiment.info', params)
+	    if code ==0:
+		for k, v in nodes.items():
+		    if v.get('erole', False) and 'pnode' in v:
+			self.node[k] = v['pnode']
+		if self.log:
+		    self.log.info("Mapping complete")
+		return True
+	    else:
+		raise service_error(service_error.internal,
+			"Cannot get node mapping of segment:%s/%s" % (pid, eid))
+
Index: fedd/Makefile/federation/experiment_control.py
===================================================================
--- fedd/Makefile/federation/experiment_control.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/experiment_control.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,2374 @@
+#!/usr/local/bin/python
+
+import os,sys
+
+import re
+import random
+import string
+import subprocess
+import tempfile
+import copy
+import pickle
+import logging
+import signal
+import time
+
+import traceback
+# For parsing visualization output and splitter output
+import xml.parsers.expat
+
+from threading import Lock, Thread, Condition
+from subprocess import call, Popen, PIPE
+
+from urlparse import urlparse
+from urllib2 import urlopen
+
+from util import *
+from fedid import fedid, generate_fedid
+from remote_service import xmlrpc_handler, soap_handler, service_caller
+from service_error import service_error
+from synch_store import synch_store
+from experiment_partition import experiment_partition
+
+import topdl
+import list_log
+from ip_allocator import ip_allocator
+from ip_addr import ip_addr
+
+
+class nullHandler(logging.Handler):
+    def emit(self, record): pass
+
+fl = logging.getLogger("fedd.experiment_control")
+fl.addHandler(nullHandler())
+
+
+# Right now, no support for composition.
+class federated_service:
+    def __init__(self, name, exporter=None, importers=None, params=None, 
+	    reqs=None, portal=None):
+	self.name=name
+	self.exporter=exporter
+	if importers is None: self.importers = []
+	else: self.importers=importers
+	if params is None: self.params = { }
+	else: self.params = params
+	if reqs is None: self.reqs = []
+	else: self.reqs = reqs
+
+	if portal is not None:
+	    self.portal = portal
+	else:
+	    self.portal = (name in federated_service.needs_portal)
+
+    def __str__(self):
+	return "name %s export %s import %s params %s reqs %s" % \
+		(self.name, self.exporter, self.importers, self.params,
+			[ (r['name'], r['visibility']) for r in self.reqs] )
+
+    needs_portal = ('SMB', 'seer', 'tmcd', 'project_export', 'seer_master')
+
+class experiment_control_local:
+    """
+    Control of experiments that this system can directly access.
+
+    Includes experiment creation, termination and information dissemination.
+    Thred safe.
+    """
+
+    class ssh_cmd_timeout(RuntimeError): pass
+    
+    class thread_pool:
+        """
+        A class to keep track of a set of threads all invoked for the same
+        task.  Manages the mutual exclusion of the states.
+        """
+        def __init__(self, nthreads):
+            """
+            Start a pool.
+            """
+            self.changed = Condition()
+            self.started = 0
+            self.terminated = 0
+            self.nthreads = nthreads
+
+        def acquire(self):
+            """
+            Get the pool's lock.
+            """
+            self.changed.acquire()
+
+        def release(self):
+            """
+            Release the pool's lock.
+            """
+            self.changed.release()
+
+        def wait(self, timeout = None):
+            """
+            Wait for a pool thread to start or stop.
+            """
+            self.changed.wait(timeout)
+
+        def start(self):
+            """
+            Called by a pool thread to report starting.
+            """
+            self.changed.acquire()
+            self.started += 1
+            self.changed.notifyAll()
+            self.changed.release()
+
+        def terminate(self):
+            """
+            Called by a pool thread to report finishing.
+            """
+            self.changed.acquire()
+            self.terminated += 1
+            self.changed.notifyAll()
+            self.changed.release()
+
+        def clear(self):
+            """
+            Clear all pool data.
+            """
+            self.changed.acquire()
+            self.started = 0
+            self.terminated =0
+            self.changed.notifyAll()
+            self.changed.release()
+
+        def wait_for_slot(self):
+            """
+            Wait until we have a free slot to start another pooled thread
+            """
+            self.acquire()
+            while self.started - self.terminated >= self.nthreads:
+                self.wait()
+            self.release()
+
+        def wait_for_all_done(self, timeout=None):
+            """
+	    Wait until all active threads finish (and at least one has
+	    started).  If a timeout is given, return after waiting that long
+	    for termination.  If all threads are done (and one has started in
+	    the since the last clear()) return True, otherwise False.
+            """
+	    if timeout:
+		deadline = time.time() + timeout
+            self.acquire()
+            while self.started == 0 or self.started > self.terminated:
+                self.wait(timeout)
+		if timeout:
+		    if time.time() > deadline:
+			break
+		    timeout = deadline - time.time()
+            self.release()
+	    return not (self.started == 0 or self.started > self.terminated)
+
+    class pooled_thread(Thread):
+        """
+        One of a set of threads dedicated to a specific task.  Uses the
+        thread_pool class above for coordination.
+        """
+        def __init__(self, group=None, target=None, name=None, args=(), 
+                kwargs={}, pdata=None, trace_file=None):
+            Thread.__init__(self, group, target, name, args, kwargs)
+            self.rv = None          # Return value of the ops in this thread
+            self.exception = None   # Exception that terminated this thread
+            self.target=target      # Target function to run on start()
+            self.args = args        # Args to pass to target
+            self.kwargs = kwargs    # Additional kw args
+            self.pdata = pdata      # thread_pool for this class
+            # Logger for this thread
+            self.log = logging.getLogger("fedd.experiment_control")
+        
+        def run(self):
+            """
+            Emulate Thread.run, except add pool data manipulation and error
+            logging.
+            """
+            if self.pdata:
+                self.pdata.start()
+
+            if self.target:
+                try:
+                    self.rv = self.target(*self.args, **self.kwargs)
+                except service_error, s:
+                    self.exception = s
+                    self.log.error("Thread exception: %s %s" % \
+                            (s.code_string(), s.desc))
+                except:
+                    self.exception = sys.exc_info()[1]
+                    self.log.error(("Unexpected thread exception: %s" +\
+                            "Trace %s") % (self.exception,\
+                                traceback.format_exc()))
+            if self.pdata:
+                self.pdata.terminate()
+
+    call_RequestAccess = service_caller('RequestAccess')
+    call_ReleaseAccess = service_caller('ReleaseAccess')
+    call_StartSegment = service_caller('StartSegment')
+    call_TerminateSegment = service_caller('TerminateSegment')
+    call_Ns2Topdl = service_caller('Ns2Topdl')
+
+    def __init__(self, config=None, auth=None):
+        """
+        Intialize the various attributes, most from the config object
+        """
+
+        def parse_tarfile_list(tf):
+            """
+            Parse a tarfile list from the configuration.  This is a set of
+            paths and tarfiles separated by spaces.
+            """
+            rv = [ ]
+            if tf is not None:
+                tl = tf.split()
+                while len(tl) > 1:
+                    p, t = tl[0:2]
+                    del tl[0:2]
+                    rv.append((p, t))
+            return rv
+
+        self.thread_with_rv = experiment_control_local.pooled_thread
+        self.thread_pool = experiment_control_local.thread_pool
+        self.list_log = list_log.list_log
+
+        self.cert_file = config.get("experiment_control", "cert_file")
+        if self.cert_file:
+            self.cert_pwd = config.get("experiment_control", "cert_pwd")
+        else:
+            self.cert_file = config.get("globals", "cert_file")
+            self.cert_pwd = config.get("globals", "cert_pwd")
+
+        self.trusted_certs = config.get("experiment_control", "trusted_certs") \
+                or config.get("globals", "trusted_certs")
+
+	self.repodir = config.get("experiment_control", "repodir")
+	self.repo_url = config.get("experiment_control", "repo_url", 
+		"https://users.isi.deterlab.net:23235");
+
+        self.exp_stem = "fed-stem"
+        self.log = logging.getLogger("fedd.experiment_control")
+        set_log_level(config, "experiment_control", self.log)
+        self.muxmax = 2
+        self.nthreads = 10
+        self.randomize_experiments = False
+
+        self.splitter = None
+        self.ssh_keygen = "/usr/bin/ssh-keygen"
+        self.ssh_identity_file = None
+
+
+        self.debug = config.getboolean("experiment_control", "create_debug")
+	self.cleanup = not config.getboolean("experiment_control", 
+		"leave_tmpfiles")
+        self.state_filename = config.get("experiment_control", 
+                "experiment_state")
+        self.store_filename = config.get("experiment_control", 
+                "synch_store")
+        self.store_url = config.get("experiment_control", "store_url")
+        self.splitter_url = config.get("experiment_control", "ns2topdl_uri")
+        self.fedkit = parse_tarfile_list(\
+                config.get("experiment_control", "fedkit"))
+        self.gatewaykit = parse_tarfile_list(\
+                config.get("experiment_control", "gatewaykit"))
+        accessdb_file = config.get("experiment_control", "accessdb")
+
+        self.ssh_pubkey_file = config.get("experiment_control", 
+                "ssh_pubkey_file")
+        self.ssh_privkey_file = config.get("experiment_control",
+                "ssh_privkey_file")
+        dt = config.get("experiment_control", "direct_transit")
+	if dt: self.direct_transit = [ tb.strip() for tb in dt.split(",")]
+	else: self.direct_transit = [ ]
+        # NB for internal master/slave ops, not experiment setup
+        self.ssh_type = config.get("experiment_control", "sshkeytype", "rsa")
+
+        self.overrides = set([])
+        ovr = config.get('experiment_control', 'overrides')
+        if ovr:
+            for o in ovr.split(","):
+                o = o.strip()
+                if o.startswith('fedid:'): o = o[len('fedid:'):]
+                self.overrides.add(fedid(hexstr=o))
+
+        self.state = { }
+        self.state_lock = Lock()
+        self.tclsh = "/usr/local/bin/otclsh"
+        self.tcl_splitter = config.get("ns2topdl", "tcl_splitter") or \
+                config.get("experiment_control", "tcl_splitter",
+                        "/usr/testbed/lib/ns2ir/parse.tcl")
+        mapdb_file = config.get("experiment_control", "mapdb")
+        self.trace_file = sys.stderr
+
+        self.def_expstart = \
+                "sudo -H /bin/sh /usr/local/federation/bin/federate.sh >& " +\
+                "/tmp/federate";
+        self.def_mexpstart = "sudo -H /usr/local/federation/bin/make_hosts " +\
+                "FEDDIR/hosts";
+        self.def_gwstart = \
+                "sudo -H /usr/local/federation/bin/fed-tun.pl -f GWCONF>& " +\
+                "/tmp/bridge.log";
+        self.def_mgwstart = \
+                "sudo -H /usr/local/federation/bin/fed-tun.pl -f GWCONF >& " +\
+                "/tmp/bridge.log";
+        self.def_gwimage = "FBSD61-TUNNEL2";
+        self.def_gwtype = "pc";
+        self.local_access = { }
+
+        if auth:
+            self.auth = auth
+        else:
+            self.log.error(\
+                    "[access]: No authorizer initialized, creating local one.")
+            auth = authorizer()
+
+
+        if self.ssh_pubkey_file:
+            try:
+                f = open(self.ssh_pubkey_file, 'r')
+                self.ssh_pubkey = f.read()
+                f.close()
+            except EnvironmentError:
+                raise service_error(service_error.internal,
+                        "Cannot read sshpubkey")
+        else:
+            raise service_error(service_error.internal, 
+                    "No SSH public key file?")
+
+        if not self.ssh_privkey_file:
+            raise service_error(service_error.internal, 
+                    "No SSH public key file?")
+
+
+        if mapdb_file:
+            self.read_mapdb(mapdb_file)
+        else:
+            self.log.warn("[experiment_control] No testbed map, using defaults")
+            self.tbmap = { 
+                    'deter':'https://users.isi.deterlab.net:23235',
+                    'emulab':'https://users.isi.deterlab.net:23236',
+                    'ucb':'https://users.isi.deterlab.net:23237',
+                    }
+
+        if accessdb_file:
+                self.read_accessdb(accessdb_file)
+        else:
+            raise service_error(service_error.internal,
+                    "No accessdb specified in config")
+
+        # Grab saved state.  OK to do this w/o locking because it's read only
+        # and only one thread should be in existence that can see self.state at
+        # this point.
+        if self.state_filename:
+            self.read_state()
+
+	if self.store_filename:
+	    self.read_store()
+	else:
+	    self.log.warning("No saved synch store")
+	    self.synch_store = synch_store
+
+        # Dispatch tables
+        self.soap_services = {\
+                'New': soap_handler('New', self.new_experiment),
+                'Create': soap_handler('Create', self.create_experiment),
+                'Vtopo': soap_handler('Vtopo', self.get_vtopo),
+                'Vis': soap_handler('Vis', self.get_vis),
+                'Info': soap_handler('Info', self.get_info),
+                'MultiInfo': soap_handler('MultiInfo', self.get_multi_info),
+                'Terminate': soap_handler('Terminate', 
+                    self.terminate_experiment),
+		'GetValue': soap_handler('GetValue', self.GetValue),
+		'SetValue': soap_handler('SetValue', self.SetValue),
+        }
+
+        self.xmlrpc_services = {\
+                'New': xmlrpc_handler('New', self.new_experiment),
+                'Create': xmlrpc_handler('Create', self.create_experiment),
+                'Vtopo': xmlrpc_handler('Vtopo', self.get_vtopo),
+                'Vis': xmlrpc_handler('Vis', self.get_vis),
+                'Info': xmlrpc_handler('Info', self.get_info),
+                'MultiInfo': xmlrpc_handler('MultiInfo', self.get_multi_info),
+                'Terminate': xmlrpc_handler('Terminate',
+                    self.terminate_experiment),
+		'GetValue': xmlrpc_handler('GetValue', self.GetValue),
+		'SetValue': xmlrpc_handler('SetValue', self.SetValue),
+        }
+
+    # Call while holding self.state_lock
+    def write_state(self):
+        """
+        Write a new copy of experiment state after copying the existing state
+        to a backup.
+
+        State format is a simple pickling of the state dictionary.
+        """
+        if os.access(self.state_filename, os.W_OK):
+            copy_file(self.state_filename, \
+		    "%s.bak" % self.state_filename)
+        try:
+            f = open(self.state_filename, 'w')
+            pickle.dump(self.state, f)
+        except EnvironmentError, e:
+            self.log.error("Can't write file %s: %s" % \
+                    (self.state_filename, e))
+        except pickle.PicklingError, e:
+            self.log.error("Pickling problem: %s" % e)
+        except TypeError, e:
+            self.log.error("Pickling problem (TypeError): %s" % e)
+
+    @staticmethod
+    def get_alloc_ids(state):
+	"""
+	Pull the fedids of the identifiers of each allocation from the
+	state.  Again, a dict dive that's best isolated.
+
+	Used by read_store and read state
+	"""
+
+	return [ f['allocID']['fedid'] 
+		for f in state.get('federant',[]) \
+		    if f.has_key('allocID') and \
+			f['allocID'].has_key('fedid')]
+
+    # Call while holding self.state_lock
+    def read_state(self):
+        """
+        Read a new copy of experiment state.  Old state is overwritten.
+
+        State format is a simple pickling of the state dictionary.
+        """
+	
+	def get_experiment_id(state):
+	    """
+	    Pull the fedid experimentID out of the saved state.  This is kind
+	    of a gross walk through the dict.
+	    """
+
+	    if state.has_key('experimentID'):
+		for e in state['experimentID']:
+		    if e.has_key('fedid'):
+			return e['fedid']
+		else:
+		    return None
+	    else:
+		return None
+
+        try:
+            f = open(self.state_filename, "r")
+            self.state = pickle.load(f)
+            self.log.debug("[read_state]: Read state from %s" % \
+                    self.state_filename)
+        except EnvironmentError, e:
+            self.log.warning("[read_state]: No saved state: Can't open %s: %s"\
+                    % (self.state_filename, e))
+        except pickle.UnpicklingError, e:
+            self.log.warning(("[read_state]: No saved state: " + \
+                    "Unpickling failed: %s") % e)
+        
+        for s in self.state.values():
+            try:
+
+		eid = get_experiment_id(s)
+		if eid : 
+		    # Give the owner rights to the experiment
+                    self.auth.set_attribute(s['owner'], eid)
+		    # And holders of the eid as well
+                    self.auth.set_attribute(eid, eid)
+                    # allow overrides to control experiments as well
+                    for o in self.overrides:
+                        self.auth.set_attribute(o, eid)
+		    # Set permissions to allow reading of the software repo, if
+		    # any, as well.
+		    for a in self.get_alloc_ids(s):
+			self.auth.set_attribute(a, 'repo/%s' % eid)
+		else:
+		    raise KeyError("No experiment id")
+            except KeyError, e:
+                self.log.warning("[read_state]: State ownership or identity " +\
+                        "misformatted in %s: %s" % (self.state_filename, e))
+
+
+    def read_accessdb(self, accessdb_file):
+        """
+        Read the mapping from fedids that can create experiments to their name
+        in the 3-level access namespace.  All will be asserted from this
+        testbed and can include the local username and porject that will be
+        asserted on their behalf by this fedd.  Each fedid is also added to the
+        authorization system with the "create" attribute.
+        """
+        self.accessdb = {}
+        # These are the regexps for parsing the db
+        name_expr = "[" + string.ascii_letters + string.digits + "\.\-]+"
+        project_line = re.compile("^\s*fedid:([" + string.hexdigits + "]+)"+ \
+                "\s*->\(\s*("+name_expr+")\s*,\s*("+name_expr+")\s*\)\s*$")
+        user_line = re.compile("^\s*fedid:([" + string.hexdigits + "]+)"+ \
+                "\s*->\s*(" + name_expr + ")\s*$")
+        lineno = 0
+
+        # Parse the mappings and store in self.authdb, a dict of 
+        # fedid -> (proj, user)
+        try:
+            f = open(accessdb_file, "r")
+            for line in f:
+                lineno += 1
+                line = line.strip()
+                if len(line) == 0 or line.startswith('#'):
+                    continue
+                m = project_line.match(line)
+                if m:
+                    fid = fedid(hexstr=m.group(1))
+                    project, user = m.group(2,3)
+                    if not self.accessdb.has_key(fid):
+                        self.accessdb[fid] = []
+                    self.accessdb[fid].append((project, user))
+                    continue
+
+                m = user_line.match(line)
+                if m:
+                    fid = fedid(hexstr=m.group(1))
+                    project = None
+                    user = m.group(2)
+                    if not self.accessdb.has_key(fid):
+                        self.accessdb[fid] = []
+                    self.accessdb[fid].append((project, user))
+                    continue
+                self.log.warn("[experiment_control] Error parsing access " +\
+                        "db %s at line %d" %  (accessdb_file, lineno))
+        except EnvironmentError:
+            raise service_error(service_error.internal,
+                    ("Error opening/reading %s as experiment " +\
+                            "control accessdb") %  accessdb_file)
+        f.close()
+
+        # Initialize the authorization attributes
+        for fid in self.accessdb.keys():
+            self.auth.set_attribute(fid, 'create')
+            self.auth.set_attribute(fid, 'new')
+
+    def read_mapdb(self, file):
+        """
+        Read a simple colon separated list of mappings for the
+        label-to-testbed-URL mappings.  Clears or creates self.tbmap.
+        """
+
+        self.tbmap = { }
+        lineno =0
+        try:
+            f = open(file, "r")
+            for line in f:
+                lineno += 1
+                line = line.strip()
+                if line.startswith('#') or len(line) == 0:
+                    continue
+                try:
+                    label, url = line.split(':', 1)
+                    self.tbmap[label] = url
+                except ValueError, e:
+                    self.log.warn("[read_mapdb] Ignored bad line (%d) in " +\
+                            "map db: %s %s" % (lineno, line, e))
+        except EnvironmentError, e:
+            self.log.warning("[read_mapdb]: No saved map database: Can't " +\
+                    "open %s: %s" % (file, e))
+        f.close()
+
+    def read_store(self):
+        try:
+	    self.synch_store = synch_store()
+	    self.synch_store.load(self.store_filename)
+	    self.log.debug("[read_store]: Read store from %s" % \
+                    self.store_filename)
+        except EnvironmentError, e:
+	    self.log.warning("[read_store]: No saved store: Can't open %s: %s"\
+                    % (self.state_filename, e))
+	    self.synch_store = synch_store()
+
+	# Set the initial permissions on data in the store.  XXX: This ad hoc
+	# authorization attribute initialization is getting out of hand.
+	for k in self.synch_store.all_keys():
+	    try:
+		if k.startswith('fedid:'):
+		    fid = fedid(hexstr=k[6:46])
+		    if self.state.has_key(fid):
+			for a in self.get_alloc_ids(self.state[fid]):
+			    self.auth.set_attribute(a, k)
+	    except ValueError, e:
+		self.log.warn("Cannot deduce permissions for %s" % k)
+
+
+    def write_store(self):
+        """
+        Write a new copy of synch_store after writing current state
+	to a backup.  We use the internal synch_store pickle method to avoid
+	incinsistent data.
+
+        State format is a simple pickling of the store.
+        """
+        if os.access(self.store_filename, os.W_OK):
+            copy_file(self.store_filename, \
+		    "%s.bak" % self.store_filename)
+        try:
+	    self.synch_store.save(self.store_filename)
+        except EnvironmentError, e:
+            self.log.error("Can't write file %s: %s" % \
+                    (self.store_filename, e))
+        except TypeError, e:
+            self.log.error("Pickling problem (TypeError): %s" % e)
+
+        
+    def generate_ssh_keys(self, dest, type="rsa" ):
+        """
+        Generate a set of keys for the gateways to use to talk.
+
+        Keys are of type type and are stored in the required dest file.
+        """
+        valid_types = ("rsa", "dsa")
+        t = type.lower();
+        if t not in valid_types: raise ValueError
+        cmd = [self.ssh_keygen, '-t', t, '-N', '', '-f', dest]
+
+        try:
+            trace = open("/dev/null", "w")
+        except EnvironmentError:
+            raise service_error(service_error.internal,
+                    "Cannot open /dev/null??");
+
+        # May raise CalledProcessError
+        self.log.debug("[generate_ssh_keys]: %s" % " ".join(cmd))
+        rv = call(cmd, stdout=trace, stderr=trace, close_fds=True)
+        if rv != 0:
+            raise service_error(service_error.internal, 
+                    "Cannot generate nonce ssh keys.  %s return code %d" \
+                            % (self.ssh_keygen, rv))
+
+    def gentopo(self, str):
+        """
+        Generate the topology dtat structure from the splitter's XML
+        representation of it.
+
+        The topology XML looks like:
+            <experiment>
+                <nodes>
+                    <node><vname></vname><ips>ip1:ip2</ips></node>
+                </nodes>
+                <lans>
+                    <lan>
+                        <vname></vname><vnode></vnode><ip></ip>
+                        <bandwidth></bandwidth><member>node:port</member>
+                    </lan>
+                </lans>
+        """
+        class topo_parse:
+            """
+            Parse the topology XML and create the dats structure.
+            """
+            def __init__(self):
+                # Typing of the subelements for data conversion
+                self.str_subelements = ('vname', 'vnode', 'ips', 'ip', 'member')
+                self.int_subelements = ( 'bandwidth',)
+                self.float_subelements = ( 'delay',)
+                # The final data structure
+                self.nodes = [ ]
+                self.lans =  [ ]
+                self.topo = { \
+                        'node': self.nodes,\
+                        'lan' : self.lans,\
+                    }
+                self.element = { }  # Current element being created
+                self.chars = ""     # Last text seen
+
+            def end_element(self, name):
+                # After each sub element the contents is added to the current
+                # element or to the appropriate list.
+                if name == 'node':
+                    self.nodes.append(self.element)
+                    self.element = { }
+                elif name == 'lan':
+                    self.lans.append(self.element)
+                    self.element = { }
+                elif name in self.str_subelements:
+                    self.element[name] = self.chars
+                    self.chars = ""
+                elif name in self.int_subelements:
+                    self.element[name] = int(self.chars)
+                    self.chars = ""
+                elif name in self.float_subelements:
+                    self.element[name] = float(self.chars)
+                    self.chars = ""
+
+            def found_chars(self, data):
+                self.chars += data.rstrip()
+
+
+        tp = topo_parse();
+        parser = xml.parsers.expat.ParserCreate()
+        parser.EndElementHandler = tp.end_element
+        parser.CharacterDataHandler = tp.found_chars
+
+        parser.Parse(str)
+
+        return tp.topo
+        
+
+    def genviz(self, topo):
+        """
+        Generate the visualization the virtual topology
+        """
+
+        neato = "/usr/local/bin/neato"
+        # These are used to parse neato output and to create the visualization
+        # file.
+        vis_re = re.compile('^\s*"?([\w\-]+)"?\s+\[.*pos="([\d\.]+),([\d\.]+)"')
+        vis_fmt = "<node><name>%s</name><x>%s</x><y>%s</y><type>" + \
+                "%s</type></node>"
+
+        try:
+            # Node names
+            nodes = [ n['vname'] for n in topo['node'] ]
+            topo_lans = topo['lan']
+        except KeyError, e:
+	    raise service_error(service_error.internal, "Bad topology: %s" %e)
+
+        lans = { }
+        links = { }
+
+        # Walk through the virtual topology, organizing the connections into
+        # 2-node connections (links) and more-than-2-node connections (lans).
+        # When a lan is created, it's added to the list of nodes (there's a
+        # node in the visualization for the lan).
+        for l in topo_lans:
+            if links.has_key(l['vname']):
+                if len(links[l['vname']]) < 2:
+                    links[l['vname']].append(l['vnode'])
+                else:
+                    nodes.append(l['vname'])
+                    lans[l['vname']] = links[l['vname']]
+                    del links[l['vname']]
+                    lans[l['vname']].append(l['vnode'])
+            elif lans.has_key(l['vname']):
+                lans[l['vname']].append(l['vnode'])
+            else:
+                links[l['vname']] = [ l['vnode'] ]
+
+
+        # Open up a temporary file for dot to turn into a visualization
+        try:
+            df, dotname = tempfile.mkstemp()
+            dotfile = os.fdopen(df, 'w')
+        except EnvironmentError:
+            raise service_error(service_error.internal,
+                    "Failed to open file in genviz")
+
+        try:
+            dnull = open('/dev/null', 'w')
+        except EnvironmentError:
+            service_error(service_error.internal,
+                    "Failed to open /dev/null in genviz")
+
+        # Generate a dot/neato input file from the links, nodes and lans
+        try:
+            print >>dotfile, "graph G {"
+            for n in nodes:
+                print >>dotfile, '\t"%s"' % n
+            for l in links.keys():
+                print >>dotfile, '\t"%s" -- "%s"' %  tuple(links[l])
+            for l in lans.keys():
+                for n in lans[l]:
+                    print >>dotfile, '\t "%s" -- "%s"' % (n,l)
+            print >>dotfile, "}"
+            dotfile.close()
+        except TypeError:
+            raise service_error(service_error.internal,
+                    "Single endpoint link in vtopo")
+        except EnvironmentError:
+            raise service_error(service_error.internal, "Cannot write dot file")
+
+        # Use dot to create a visualization
+        dot = Popen([neato, '-Gstart=rand', '-Gepsilon=0.005', '-Gmaxiter=2000',
+                '-Gpack=true', dotname], stdout=PIPE, stderr=dnull,
+                close_fds=True)
+        dnull.close()
+
+        # Translate dot to vis format
+        vis_nodes = [ ]
+        vis = { 'node': vis_nodes }
+        for line in dot.stdout:
+            m = vis_re.match(line)
+            if m:
+                vn = m.group(1)
+                vis_node = {'name': vn, \
+                        'x': float(m.group(2)),\
+                        'y' : float(m.group(3)),\
+                    }
+                if vn in links.keys() or vn in lans.keys():
+                    vis_node['type'] = 'lan'
+                else:
+                    vis_node['type'] = 'node'
+                vis_nodes.append(vis_node)
+        rv = dot.wait()
+
+        os.remove(dotname)
+        if rv == 0 : return vis
+        else: return None
+
+    def get_access(self, tb, nodes, tbparam, access_user, masters, tbmap):
+        """
+        Get access to testbed through fedd and set the parameters for that tb
+        """
+	def get_export_project(svcs):
+	    """
+	    Look through for the list of federated_service for this testbed
+	    objects for a project_export service, and extract the project
+	    parameter.
+	    """
+
+	    pe = [s for s in svcs if s.name=='project_export']
+	    if len(pe) == 1:
+		return pe[0].params.get('project', None)
+	    elif len(pe) == 0:
+		return None
+	    else:
+		raise service_error(service_error.req,
+			"More than one project export is not supported")
+
+        uri = tbmap.get(testbed_base(tb), None)
+        if not uri:
+            raise service_error(service_error.server_config, 
+                    "Unknown testbed: %s" % tb)
+
+	export_svcs = masters.get(tb,[])
+	import_svcs = [ s for m in masters.values() \
+		for s in m \
+		    if tb in s.importers ]
+
+	export_project = get_export_project(export_svcs)
+
+        # Tweak search order so that if there are entries in access_user that 
+	# have a project matching the export project, we try them first 
+	if export_project: 
+	    access_sequence = [ (p, u) for p, u in access_user \
+		    if p == export_project] 
+	    access_sequence.extend([(p, u) for p, u in access_user \
+		    if p != export_project]) 
+	else: 
+	    access_sequence = access_user 
+
+	for p, u in access_sequence: 
+            self.log.debug(("[get_access] Attempting access from (%s, %s) " + \
+                    "to %s") %  ((p or "None"), u, uri))
+
+            if p:
+                # Request with user and project specified
+                req = {\
+			'credential': [ "project: %s" % p, "user: %s"  % u],
+                    }
+            else:
+                # Request with only user specified
+                req = {\
+			'credential': [ 'user: %s' % u ],
+                    }
+
+	    # Make the service request from the services we're importing and
+	    # exporting.  Keep track of the export request ids so we can
+	    # collect the resulting info from the access response.
+	    e_keys = { }
+	    if import_svcs or export_svcs:
+		req['service'] = [ ]
+
+		for i, s in enumerate(import_svcs):
+		    idx = 'import%d' % i
+		    sr = {'id': idx, 'name': s.name, 'visibility': 'import' }
+		    if s.params:
+			sr['fedAttr'] = [ { 'attribute': k, 'value': v } \
+				for k, v in s.params.items()]
+		    req['service'].append(sr)
+
+		for i, s in enumerate(export_svcs):
+		    idx = 'export%d' % i
+		    e_keys[idx] = s
+		    sr = {'id': idx, 'name': s.name, 'visibility': 'export' }
+		    if s.params:
+			sr['fedAttr'] = [ { 'attribute': k, 'value': v } 
+				for k, v in s.params.items()]
+		    req['service'].append(sr)
+
+            # node resources if any
+            if nodes != None and len(nodes) > 0:
+                rnodes = [ ]
+                for n in nodes:
+                    rn = { }
+                    image, hw, count = n.split(":")
+                    if image: rn['image'] = [ image ]
+                    if hw: rn['hardware'] = [ hw ]
+                    if count and int(count) >0 : rn['count'] = int(count)
+                    rnodes.append(rn)
+                req['resources']= { }
+                req['resources']['node'] = rnodes
+
+            try:
+                if self.local_access.has_key(uri):
+                    # Local access call
+                    req = { 'RequestAccessRequestBody' : req }
+                    r = self.local_access[uri].RequestAccess(req, 
+                            fedid(file=self.cert_file))
+                    r = { 'RequestAccessResponseBody' : r }
+                else:
+                    r = self.call_RequestAccess(uri, req, 
+                            self.cert_file, self.cert_pwd, self.trusted_certs)
+            except service_error, e:
+                if e.code == service_error.access:
+                    self.log.debug("[get_access] Access denied")
+                    r = None
+                    continue
+                else:
+                    raise e
+
+            if r.has_key('RequestAccessResponseBody'):
+                # Through to here we have a valid response, not a fault.
+                # Access denied is a fault, so something better or worse than
+                # access denied has happened.
+                r = r['RequestAccessResponseBody']
+                self.log.debug("[get_access] Access granted")
+                break
+            else:
+                raise service_error(service_error.protocol,
+                        "Bad proxy response")
+        
+        if not r:
+            raise service_error(service_error.access, 
+                    "Access denied by %s (%s)" % (tb, uri))
+
+	tbparam[tb] = { 
+		"allocID" : r['allocID'],
+		"uri": uri,
+		}
+
+	# Collect the responses corresponding to the services this testbed
+	# exports.  These will be the service requests that we will include in
+	# the start segment requests (with appropriate visibility values) to
+	# import and export the segments.
+	for s in r.get('service', []):
+	    id = s.get('id', None)
+	    if id and id in e_keys:
+		e_keys[id].reqs.append(s)
+
+	# Add attributes to parameter space.  We don't allow attributes to
+	# overlay any parameters already installed.
+	for a in r.get('fedAttr', []):
+	    try:
+		if a['attribute'] and \
+			isinstance(a['attribute'], basestring)\
+			and not tbparam[tb].has_key(a['attribute'].lower()):
+		    tbparam[tb][a['attribute'].lower()] = a['value']
+	    except KeyError:
+		self.log.error("Bad attribute in response: %s" % a)
+
+    def release_access(self, tb, aid, tbmap=None, uri=None):
+        """
+        Release access to testbed through fedd
+        """
+
+	if not uri and tbmap:
+	    uri = tbmap.get(tb, None)
+        if not uri:
+            raise service_error(service_error.server_config, 
+                    "Unknown testbed: %s" % tb)
+
+        if self.local_access.has_key(uri):
+            resp = self.local_access[uri].ReleaseAccess(\
+                    { 'ReleaseAccessRequestBody' : {'allocID': aid},}, 
+                    fedid(file=self.cert_file))
+            resp = { 'ReleaseAccessResponseBody': resp } 
+        else:
+            resp = self.call_ReleaseAccess(uri, {'allocID': aid},
+                    self.cert_file, self.cert_pwd, self.trusted_certs)
+
+        # better error coding
+
+    def remote_ns2topdl(self, uri, desc):
+
+        req = {
+                'description' : { 'ns2description': desc },
+            }
+
+        r = self.call_Ns2Topdl(uri, req, self.cert_file, self.cert_pwd, 
+                self.trusted_certs)
+
+        if r.has_key('Ns2TopdlResponseBody'):
+            r = r['Ns2TopdlResponseBody']
+	    ed = r.get('experimentdescription', None)
+            if ed.has_key('topdldescription'):
+                return topdl.Topology(**ed['topdldescription'])
+            else:
+                raise service_error(service_error.protocol, 
+                        "Bad splitter response (no output)")
+        else:
+            raise service_error(service_error.protocol, "Bad splitter response")
+
+    class start_segment:
+	def __init__(self, debug=False, log=None, testbed="", cert_file=None,
+		cert_pwd=None, trusted_certs=None, caller=None,
+		log_collector=None):
+	    self.log = log
+	    self.debug = debug
+	    self.cert_file = cert_file
+	    self.cert_pwd = cert_pwd
+	    self.trusted_certs = None
+	    self.caller = caller
+	    self.testbed = testbed
+	    self.log_collector = log_collector
+	    self.response = None
+	    self.node = { }
+
+	def make_map(self, resp):
+	    for e in resp.get('embedding', []):
+		if 'toponame' in e and 'physname' in e:
+		    self.node[e['toponame']] = e['physname'][0]
+
+	def __call__(self, uri, aid, topo, masters, attrs=None, connInfo=None):
+	    req = {
+		    'allocID': { 'fedid' : aid }, 
+		    'segmentdescription': { 
+			'topdldescription': topo.to_dict(),
+		    },
+		}
+
+	    if connInfo:
+		req['connection'] = connInfo
+
+	    import_svcs = [ s for m in masters.values() \
+		    for s in m if self.testbed in s.importers]
+
+	    if import_svcs or self.testbed in masters:
+		req['service'] = []
+
+	    for s in import_svcs:
+		for r in s.reqs:
+		    sr = copy.deepcopy(r)
+		    sr['visibility'] = 'import';
+		    req['service'].append(sr)
+
+	    for s in masters.get(self.testbed, []):
+		for r in s.reqs:
+		    sr = copy.deepcopy(r)
+		    sr['visibility'] = 'export';
+		    req['service'].append(sr)
+
+	    if attrs:
+		req['fedAttr'] = attrs
+
+	    try:
+		self.log.debug("Calling StartSegment at %s " % uri)
+		r = self.caller(uri, req, self.cert_file, self.cert_pwd,
+			self.trusted_certs)
+		if r.has_key('StartSegmentResponseBody'):
+		    lval = r['StartSegmentResponseBody'].get('allocationLog',
+			    None)
+		    if lval and self.log_collector:
+			for line in  lval.splitlines(True):
+			    self.log_collector.write(line)
+		    self.make_map(r['StartSegmentResponseBody'])
+		    self.response = r
+		else:
+		    raise service_error(service_error.internal, 
+			    "Bad response!?: %s" %r)
+		return True
+	    except service_error, e:
+		self.log.error("Start segment failed on %s: %s" % \
+			(self.testbed, e))
+		return False
+
+
+
+    class terminate_segment:
+	def __init__(self, debug=False, log=None, testbed="", cert_file=None,
+		cert_pwd=None, trusted_certs=None, caller=None):
+	    self.log = log
+	    self.debug = debug
+	    self.cert_file = cert_file
+	    self.cert_pwd = cert_pwd
+	    self.trusted_certs = None
+	    self.caller = caller
+	    self.testbed = testbed
+
+	def __call__(self, uri, aid ):
+	    req = {
+		    'allocID': aid , 
+		}
+	    try:
+		r = self.caller(uri, req, self.cert_file, self.cert_pwd,
+			self.trusted_certs)
+		return True
+	    except service_error, e:
+		self.log.error("Terminate segment failed on %s: %s" % \
+			(self.testbed, e))
+		return False
+    
+
+    def allocate_resources(self, allocated, masters, eid, expid, 
+            tbparams, top, topo, tmpdir, alloc_log=None, log_collector=None, 
+	    attrs=None, connInfo={}, tbmap=None):
+
+        started = { }           # Testbeds where a sub-experiment started 
+                                # successfully
+
+        # XXX
+        fail_soft = False
+
+	if tbmap is None: tbmap = { }
+
+        log = alloc_log or self.log
+
+        thread_pool = self.thread_pool(self.nthreads)
+        threads = [ ]
+	starters = [ ]
+
+	for tb in allocated.keys():
+	    # Create and start a thread to start the segment, and save it
+	    # to get the return value later
+	    tb_attrs = copy.copy(attrs)
+	    thread_pool.wait_for_slot()
+	    uri = tbparams[tb].get('uri', tbmap.get(testbed_base(tb), None))
+	    base, suffix = split_testbed(tb)
+	    if suffix:
+		tb_attrs.append({'attribute': 'experiment_name', 
+		    'value': "%s-%s" % (eid, suffix)})
+	    else:
+		tb_attrs.append({'attribute': 'experiment_name', 'value': eid})
+	    if not uri:
+		raise service_error(service_error.internal, 
+			"Unknown testbed %s !?" % tb)
+
+	    if tbparams[tb].has_key('allocID') and \
+		    tbparams[tb]['allocID'].has_key('fedid'):
+		aid = tbparams[tb]['allocID']['fedid']
+	    else:
+		raise service_error(service_error.internal, 
+			"No alloc id for testbed %s !?" % tb)
+
+	    s = self.start_segment(log=log, debug=self.debug,
+		    testbed=tb, cert_file=self.cert_file,
+		    cert_pwd=self.cert_pwd, trusted_certs=self.trusted_certs,
+		    caller=self.call_StartSegment,
+		    log_collector=log_collector)
+	    starters.append(s)
+	    t  = self.pooled_thread(\
+		    target=s, name=tb,
+		    args=(uri, aid, topo[tb], masters, tb_attrs, connInfo[tb]),
+		    pdata=thread_pool, trace_file=self.trace_file)
+	    threads.append(t)
+	    t.start()
+
+	# Wait until all finish (keep pinging the log, though)
+	mins = 0
+	revoked = False
+	while not thread_pool.wait_for_all_done(60.0):
+	    mins += 1
+	    alloc_log.info("Waiting for sub threads (it has been %d mins)" \
+		    % mins)
+	    if not revoked and \
+		    len([ t.getName() for t in threads if t.rv == False]) > 0:
+		# a testbed has failed.  Revoke this experiment's
+		# synchronizarion values so that sub experiments will not
+		# deadlock waiting for synchronization that will never happen
+		self.log.info("A subexperiment has failed to swap in, " + \
+			"revoking synch keys")
+		var_key = "fedid:%s" % expid
+		for k in self.synch_store.all_keys():
+		    if len(k) > 45 and k[0:46] == var_key:
+			self.synch_store.revoke_key(k)
+		revoked = True
+
+        failed = [ t.getName() for t in threads if not t.rv ]
+        succeeded = [tb for tb in allocated.keys() if tb not in failed]
+
+        # If one failed clean up, unless fail_soft is set
+        if failed:
+            if not fail_soft:
+                thread_pool.clear()
+                for tb in succeeded:
+                    # Create and start a thread to stop the segment
+                    thread_pool.wait_for_slot()
+		    uri = tbparams[tb]['uri']
+                    t  = self.pooled_thread(\
+                            target=self.terminate_segment(log=log,
+				testbed=tb,
+				cert_file=self.cert_file, 
+				cert_pwd=self.cert_pwd,
+				trusted_certs=self.trusted_certs,
+				caller=self.call_TerminateSegment),
+                            args=(uri, tbparams[tb]['federant']['allocID']),
+			    name=tb,
+                            pdata=thread_pool, trace_file=self.trace_file)
+                    t.start()
+                # Wait until all finish (if any are being stopped)
+		if succeeded:
+		    thread_pool.wait_for_all_done()
+
+                # release the allocations
+                for tb in tbparams.keys():
+                    self.release_access(tb, tbparams[tb]['allocID'], 
+			    tbmap=tbmap, uri=tbparams[tb].get('uri', None))
+                # Remove the placeholder
+                self.state_lock.acquire()
+                self.state[eid]['experimentStatus'] = 'failed'
+                if self.state_filename: self.write_state()
+                self.state_lock.release()
+		# Remove the repo dir
+		self.remove_dirs("%s/%s" %(self.repodir, expid))
+		# Walk up tmpdir, deleting as we go
+		if self.cleanup:
+		    self.remove_dirs(tmpdir)
+		else:
+		    log.debug("[start_experiment]: not removing %s" % tmpdir)
+
+
+                log.error("Swap in failed on %s" % ",".join(failed))
+                return
+        else:
+	    # Walk through the successes and gather the virtual to physical
+	    # mapping.
+	    embedding = [ ]
+	    for s in starters:
+		for k, v in s.node.items():
+		    embedding.append({
+			'toponame': k, 
+			'physname': [ v],
+			'testbed': s.testbed
+			})
+            log.info("[start_segment]: Experiment %s active" % eid)
+
+
+        # Walk up tmpdir, deleting as we go
+	if self.cleanup:
+	    self.remove_dirs(tmpdir)
+	else:
+	    log.debug("[start_experiment]: not removing %s" % tmpdir)
+
+        # Insert the experiment into our state and update the disk copy.
+        self.state_lock.acquire()
+        self.state[expid]['experimentStatus'] = 'active'
+        self.state[eid] = self.state[expid]
+	self.state[eid]['experimentdescription']['topdldescription'] = \
+		top.to_dict()
+	self.state[eid]['embedding'] = embedding
+        if self.state_filename: self.write_state()
+        self.state_lock.release()
+        return
+
+
+    def add_kit(self, e, kit):
+	"""
+	Add a Software object created from the list of (install, location)
+	tuples passed as kit  to the software attribute of an object e.  We
+	do this enough to break out the code, but it's kind of a hack to
+	avoid changing the old tuple rep.
+	"""
+
+	s = [ topdl.Software(install=i, location=l) for i, l in kit]
+
+	if isinstance(e.software, list): e.software.extend(s)
+	else: e.software = s 
+
+
+    def create_experiment_state(self, fid, req, expid, expcert,
+	    state='starting'):
+	"""
+	Create the initial entry in the experiment's state.  The expid and
+	expcert are the experiment's fedid and certifacte that represents that
+	ID, which are installed in the experiment state.  If the request
+	includes a suggested local name that is used if possible.  If the local
+	name is already taken by an experiment owned by this user that has
+	failed, it is overwritten.  Otherwise new letters are added until a
+	valid localname is found.  The generated local name is returned.
+	"""
+
+        if req.has_key('experimentID') and \
+                req['experimentID'].has_key('localname'):
+            overwrite = False
+            eid = req['experimentID']['localname']
+            # If there's an old failed experiment here with the same local name
+            # and accessible by this user, we'll overwrite it, otherwise we'll
+            # fall through and do the collision avoidance.
+            old_expid = self.get_experiment_fedid(eid)
+            if old_expid and self.check_experiment_access(fid, old_expid):
+                self.state_lock.acquire()
+                status = self.state[eid].get('experimentStatus', None)
+                if status and status == 'failed':
+                    # remove the old access attribute
+                    self.auth.unset_attribute(fid, old_expid)
+                    overwrite = True
+                    del self.state[eid]
+                    del self.state[old_expid]
+                self.state_lock.release()
+            self.state_lock.acquire()
+            while (self.state.has_key(eid) and not overwrite):
+                eid += random.choice(string.ascii_letters)
+            # Initial state
+            self.state[eid] = {
+                    'experimentID' : \
+                            [ { 'localname' : eid }, {'fedid': expid } ],
+                    'experimentStatus': state,
+                    'experimentAccess': { 'X509' : expcert },
+                    'owner': fid,
+                    'log' : [],
+                }
+            self.state[expid] = self.state[eid]
+            if self.state_filename: self.write_state()
+            self.state_lock.release()
+        else:
+            eid = self.exp_stem
+            for i in range(0,5):
+                eid += random.choice(string.ascii_letters)
+            self.state_lock.acquire()
+            while (self.state.has_key(eid)):
+                eid = self.exp_stem
+                for i in range(0,5):
+                    eid += random.choice(string.ascii_letters)
+            # Initial state
+            self.state[eid] = {
+                    'experimentID' : \
+                            [ { 'localname' : eid }, {'fedid': expid } ],
+                    'experimentStatus': state,
+                    'experimentAccess': { 'X509' : expcert },
+                    'owner': fid,
+                    'log' : [],
+                }
+            self.state[expid] = self.state[eid]
+            if self.state_filename: self.write_state()
+            self.state_lock.release()
+
+	return eid
+
+
+    def allocate_ips_to_topo(self, top):
+	"""
+	Add an ip4_address attribute to all the hosts in the topology, based on
+	the shared substrates on which they sit.  An /etc/hosts file is also
+	created and returned as a list of hostfiles entries.  We also return
+	the allocator, because we may need to allocate IPs to portals
+	(specifically DRAGON portals).
+	"""
+	subs = sorted(top.substrates, 
+		cmp=lambda x,y: cmp(len(x.interfaces), len(y.interfaces)),
+		reverse=True)
+	ips = ip_allocator(int(ip_addr("10.0.0.0")), 2 **24)
+	ifs = { }
+	hosts = [ ]
+
+	for idx, s in enumerate(subs):
+	    net_size = len(s.interfaces)+2
+
+	    a = ips.allocate(net_size)
+	    if a :
+		base, num = a
+		if num < net_size: 
+		    raise service_error(service_error.internal,
+			    "Allocator returned wrong number of IPs??")
+	    else:
+		raise service_error(service_error.req, 
+			"Cannot allocate IP addresses")
+	    mask = ips.min_alloc
+	    while mask < net_size:
+		mask *= 2
+
+	    netmask = ((2**32-1) ^ (mask-1))
+
+	    base += 1
+	    for i in s.interfaces:
+		i.attribute.append(
+			topdl.Attribute('ip4_address', 
+			    "%s" % ip_addr(base)))
+		i.attribute.append(
+			topdl.Attribute('ip4_netmask', 
+			    "%s" % ip_addr(int(netmask))))
+
+		hname = i.element.name
+		if ifs.has_key(hname):
+		    hosts.append("%s\t%s-%s %s-%d" % \
+			    (ip_addr(base), hname, s.name, hname,
+				ifs[hname]))
+		else:
+		    ifs[hname] = 0
+		    hosts.append("%s\t%s-%s %s-%d %s" % \
+			    (ip_addr(base), hname, s.name, hname,
+				ifs[hname], hname))
+
+		ifs[hname] += 1
+		base += 1
+	return hosts, ips
+
+    def get_access_to_testbeds(self, testbeds, access_user, allocated, 
+	    tbparams, masters, tbmap):
+	"""
+	Request access to the various testbeds required for this instantiation
+	(passed in as testbeds).  User, access_user, expoert_project and master
+	are used to construct the correct requests.  Per-testbed parameters are
+	returned in tbparams.
+	"""
+	for tb in testbeds:
+	    self.get_access(tb, None, tbparams, access_user, masters, tbmap)
+	    allocated[tb] = 1
+
+    def split_topology(self, top, topo, testbeds):
+	"""
+	Create the sub-topologies that are needed for experiment instantiation.
+	"""
+	for tb in testbeds:
+	    topo[tb] = top.clone()
+	    # copy in for loop allows deletions from the original
+	    for e in [ e for e in topo[tb].elements]:
+		etb = e.get_attribute('testbed')
+		# NB: elements without a testbed attribute won't appear in any
+		# sub topologies.  
+		if not etb or etb != tb:
+		    for i in e.interface:
+			for s in i.subs:
+			    try:
+				s.interfaces.remove(i)
+			    except ValueError:
+				raise service_error(service_error.internal,
+					"Can't remove interface??")
+		    topo[tb].elements.remove(e)
+	    topo[tb].make_indices()
+
+    def wrangle_software(self, expid, top, topo, tbparams):
+	"""
+	Copy software out to the repository directory, allocate permissions and
+	rewrite the segment topologies to look for the software in local
+	places.
+	"""
+
+	# Copy the rpms and tarfiles to a distribution directory from
+	# which the federants can retrieve them
+	linkpath = "%s/software" %  expid
+	softdir ="%s/%s" % ( self.repodir, linkpath)
+	softmap = { }
+	# These are in a list of tuples format (each kit).  This comprehension
+	# unwraps them into a single list of tuples that initilaizes the set of
+	# tuples.
+	pkgs = set([ t for l in [self.fedkit, self.gatewaykit] \
+		for p, t in l ])
+	pkgs.update([x.location for e in top.elements \
+		for x in e.software])
+	try:
+	    os.makedirs(softdir)
+	except EnvironmentError, e:
+	    raise service_error(
+		    "Cannot create software directory: %s" % e)
+	# The actual copying.  Everything's converted into a url for copying.
+	for pkg in pkgs:
+	    loc = pkg
+
+	    scheme, host, path = urlparse(loc)[0:3]
+	    dest = os.path.basename(path)
+	    if not scheme:
+		if not loc.startswith('/'):
+		    loc = "/%s" % loc
+		loc = "file://%s" %loc
+	    try:
+		u = urlopen(loc)
+	    except Exception, e:
+		raise service_error(service_error.req, 
+			"Cannot open %s: %s" % (loc, e))
+	    try:
+		f = open("%s/%s" % (softdir, dest) , "w")
+		self.log.debug("Writing %s/%s" % (softdir,dest) )
+		data = u.read(4096)
+		while data:
+		    f.write(data)
+		    data = u.read(4096)
+		f.close()
+		u.close()
+	    except Exception, e:
+		raise service_error(service_error.internal,
+			"Could not copy %s: %s" % (loc, e))
+	    path = re.sub("/tmp", "", linkpath)
+	    # XXX
+	    softmap[pkg] = \
+		    "%s/%s/%s" %\
+		    ( self.repo_url, path, dest)
+
+	    # Allow the individual segments to access the software.
+	    for tb in tbparams.keys():
+		self.auth.set_attribute(tbparams[tb]['allocID']['fedid'], 
+			"/%s/%s" % ( path, dest))
+
+	# Convert the software locations in the segments into the local
+	# copies on this host
+	for soft in [ s for tb in topo.values() \
+		for e in tb.elements \
+		    if getattr(e, 'software', False) \
+			for s in e.software ]:
+	    if softmap.has_key(soft.location):
+		soft.location = softmap[soft.location]
+
+
+    def new_experiment(self, req, fid):
+        """
+	The external interface to empty initial experiment creation called from
+	the dispatcher.
+
+        Creates a working directory, splits the incoming description using the
+        splitter script and parses out the avrious subsections using the
+        lcasses above.  Once each sub-experiment is created, use pooled threads
+        to instantiate them and start it all up.
+        """
+        if not self.auth.check_attribute(fid, 'new'):
+            raise service_error(service_error.access, "New access denied")
+
+        try:
+            tmpdir = tempfile.mkdtemp(prefix="split-")
+        except EnvironmentError:
+            raise service_error(service_error.internal, "Cannot create tmp dir")
+
+        try:
+            access_user = self.accessdb[fid]
+        except KeyError:
+            raise service_error(service_error.internal,
+                    "Access map and authorizer out of sync in " + \
+                            "new_experiment for fedid %s"  % fid)
+
+        pid = "dummy"
+        gid = "dummy"
+
+        req = req.get('NewRequestBody', None)
+        if not req:
+            raise service_error(service_error.req,
+                    "Bad request format (no NewRequestBody)")
+
+        # Generate an ID for the experiment (slice) and a certificate that the
+        # allocator can use to prove they own it.  We'll ship it back through
+        # the encrypted connection.
+        (expid, expcert) = generate_fedid("test", dir=tmpdir, log=self.log)
+
+	#now we're done with the tmpdir, and it should be empty
+	if self.cleanup:
+	    self.log.debug("[new_experiment]: removing %s" % tmpdir)
+	    os.rmdir(tmpdir)
+	else:
+	    self.log.debug("[new_experiment]: not removing %s" % tmpdir)
+
+	eid = self.create_experiment_state(fid, req, expid, expcert, 
+		state='empty')
+
+        # Let users touch the state
+        self.auth.set_attribute(fid, expid)
+        self.auth.set_attribute(expid, expid)
+        # Override fedids can manipulate state as well
+        for o in self.overrides:
+            self.auth.set_attribute(o, expid)
+
+        rv = {
+                'experimentID': [
+                    {'localname' : eid }, { 'fedid': copy.copy(expid) }
+                ],
+                'experimentStatus': 'empty',
+                'experimentAccess': { 'X509' : expcert }
+            }
+
+        return rv
+
+    def create_experiment(self, req, fid):
+        """
+        The external interface to experiment creation called from the
+        dispatcher.
+
+        Creates a working directory, splits the incoming description using the
+        splitter script and parses out the various subsections using the
+	classes above.  Once each sub-experiment is created, use pooled threads
+	to instantiate them and start it all up.
+        """
+
+        req = req.get('CreateRequestBody', None)
+        if not req:
+            raise service_error(service_error.req,
+                    "Bad request format (no CreateRequestBody)")
+
+        # Get the experiment access
+        exp = req.get('experimentID', None)
+        if exp:
+            if exp.has_key('fedid'):
+                key = exp['fedid']
+		expid = key
+		eid = None
+            elif exp.has_key('localname'):
+                key = exp['localname']
+		eid = key
+		expid = None
+            else:
+                raise service_error(service_error.req, "Unknown lookup type")
+        else:
+            raise service_error(service_error.req, "No request?")
+
+        self.check_experiment_access(fid, key)
+
+	# Install the testbed map entries supplied with the request into a copy
+	# of the testbed map.
+	tbmap = dict(self.tbmap)
+	for m in req.get('testbedmap', []):
+	    if 'testbed' in m and 'uri' in m:
+		tbmap[m['testbed']] = m['uri']
+
+        try:
+            tmpdir = tempfile.mkdtemp(prefix="split-")
+            os.mkdir(tmpdir+"/keys")
+        except EnvironmentError:
+            raise service_error(service_error.internal, "Cannot create tmp dir")
+
+        gw_pubkey_base = "fed.%s.pub" % self.ssh_type
+        gw_secretkey_base = "fed.%s" % self.ssh_type
+        gw_pubkey = tmpdir + "/keys/" + gw_pubkey_base
+        gw_secretkey = tmpdir + "/keys/" + gw_secretkey_base
+        tclfile = tmpdir + "/experiment.tcl"
+        tbparams = { }
+        try:
+            access_user = self.accessdb[fid]
+        except KeyError:
+            raise service_error(service_error.internal,
+                    "Access map and authorizer out of sync in " + \
+                            "create_experiment for fedid %s"  % fid)
+
+        pid = "dummy"
+        gid = "dummy"
+
+        # The tcl parser needs to read a file so put the content into that file
+        descr=req.get('experimentdescription', None)
+        if descr:
+            file_content=descr.get('ns2description', None)
+            if file_content:
+                try:
+                    f = open(tclfile, 'w')
+                    f.write(file_content)
+                    f.close()
+                except EnvironmentError:
+                    raise service_error(service_error.internal,
+                            "Cannot write temp experiment description")
+            else:
+                raise service_error(service_error.req, 
+                        "Only ns2descriptions supported")
+        else:
+            raise service_error(service_error.req, "No experiment description")
+
+	self.state_lock.acquire()
+	if self.state.has_key(key):
+	    self.state[key]['experimentStatus'] = "starting"
+	    for e in self.state[key].get('experimentID',[]):
+		if not expid and e.has_key('fedid'):
+		    expid = e['fedid']
+		elif not eid and e.has_key('localname'):
+		    eid = e['localname']
+	self.state_lock.release()
+
+	if not (eid and expid):
+	    raise service_error(service_error.internal, 
+		    "Cannot find local experiment info!?")
+
+        try: 
+            # This catches exceptions to clear the placeholder if necessary
+            try:
+                self.generate_ssh_keys(gw_secretkey, self.ssh_type)
+            except ValueError:
+                raise service_error(service_error.server_config, 
+                        "Bad key type (%s)" % self.ssh_type)
+
+	    # Copy the service request
+	    tb_services = [ s for s in req.get('service',[]) ]
+	    # Translate to topdl
+            if self.splitter_url:
+                self.log.debug("Calling remote topdl translator at %s" % \
+                        self.splitter_url)
+                top = self.remote_ns2topdl(self.splitter_url, file_content)
+            else:
+                tclcmd = [self.tclsh, self.tcl_splitter, '-t', '-x', 
+                    str(self.muxmax), '-m', 'dummy']
+
+                tclcmd.extend([pid, gid, eid, tclfile])
+
+                self.log.debug("running local splitter %s", " ".join(tclcmd))
+                # This is just fantastic.  As a side effect the parser copies
+                # tb_compat.tcl into the current directory, so that directory
+                # must be writable by the fedd user.  Doing this in the
+                # temporary subdir ensures this is the case.
+                tclparser = Popen(tclcmd, stdout=PIPE, close_fds=True, 
+                        cwd=tmpdir)
+                split_data = tclparser.stdout
+
+		top = topdl.topology_from_xml(file=split_data, top="experiment")
+
+	    hosts, ip_allocator = self.allocate_ips_to_topo(top)
+	    # Find the testbeds to look up
+            testbeds = set([ a.value for e in top.elements \
+		    for a in e.attribute \
+			if a.attribute == 'testbed'])
+
+	    tb_hosts = { }
+	    for tb in testbeds:
+		tb_hosts[tb] = [ e.name for e in top.elements \
+			if isinstance(e, topdl.Computer) and \
+			    e.get_attribute('testbed') and \
+			    e.get_attribute('testbed') == tb]
+
+	    masters = { }	    # testbeds exporting services
+	    pmasters = { }	    # Testbeds exporting services that 
+				    # need portals
+	    for s in tb_services:
+		# If this is a service request with the importall field
+		# set, fill it out.
+
+		if s.get('importall', False):
+		    s['import'] = [ tb for tb in testbeds \
+			    if tb not in s.get('export',[])]
+		    del s['importall']
+
+		# Add the service to masters
+		for tb in s.get('export', []):
+		    if s.get('name', None):
+			if tb not in masters:
+			    masters[tb] = [ ]
+
+			params = { }
+			if 'fedAttr' in s:
+			    for a in s['fedAttr']:
+				params[a.get('attribute', '')] = \
+					a.get('value','')
+
+			fser = federated_service(name=s['name'],
+				exporter=tb, importers=s.get('import',[]),
+				params=params)
+			if fser.name == 'hide_hosts' \
+				and 'hosts' not in fser.params:
+			    fser.params['hosts'] = \
+				    ",".join(tb_hosts.get(fser.exporter, []))
+			masters[tb].append(fser)
+
+			if fser.portal:
+			    if tb not in pmasters: pmasters[tb] = [ fser ]
+			    else: pmasters[tb].append(fser)
+		    else:
+			self.log.error('Testbed service does not have name " + \
+				"and importers')
+
+
+            allocated = { }         # Testbeds we can access
+	    topo ={ }		    # Sub topologies
+	    connInfo = { }	    # Connection information
+
+	    self.get_access_to_testbeds(testbeds, access_user, allocated, 
+		    tbparams, masters, tbmap)
+
+	    self.split_topology(top, topo, testbeds)
+
+	    # Copy configuration files into the remote file store
+	    # The config urlpath
+	    configpath = "/%s/config" % expid
+	    # The config file system location
+	    configdir ="%s%s" % ( self.repodir, configpath)
+	    try:
+		os.makedirs(configdir)
+	    except EnvironmentError, e:
+		raise service_error(service_error.internal,
+			"Cannot create config directory: %s" % e)
+	    try:
+		f = open("%s/hosts" % configdir, "w")
+		f.write('\n'.join(hosts))
+		f.close()
+	    except EnvironmentError, e:
+		raise service_error(service_error.internal, 
+			"Cannot write hosts file: %s" % e)
+	    try:
+		copy_file("%s" % gw_pubkey, "%s/%s" % \
+			(configdir, gw_pubkey_base))
+		copy_file("%s" % gw_secretkey, "%s/%s" % \
+			(configdir, gw_secretkey_base))
+	    except EnvironmentError, e:
+		raise service_error(service_error.internal, 
+			"Cannot copy keyfiles: %s" % e)
+
+	    # Allow the individual testbeds to access the configuration files.
+	    for tb in tbparams.keys():
+		asignee = tbparams[tb]['allocID']['fedid']
+		for f in ("hosts", gw_secretkey_base, gw_pubkey_base):
+		    self.auth.set_attribute(asignee, "%s/%s" % (configpath, f))
+
+	    part = experiment_partition(self.auth, self.store_url, tbmap,
+		    self.muxmax, self.direct_transit)
+	    part.add_portals(top, topo, eid, pmasters, tbparams, ip_allocator,
+		    connInfo, expid)
+	    # Now get access to the dynamic testbeds (those added above)
+	    for tb in [ t for t in topo if t not in allocated]:
+		self.get_access(tb, None, tbparams, access_user, masters)
+		allocated[tb] = 1
+		store_keys = topo[tb].get_attribute('store_keys')
+		# Give the testbed access to keys it exports or imports
+		if store_keys:
+		    for sk in store_keys.split(" "):
+			self.auth.set_attribute(\
+				tbparams[tb]['allocID']['fedid'], sk)
+
+	    self.wrangle_software(expid, top, topo, tbparams)
+
+	    vtopo = topdl.topology_to_vtopo(top)
+	    vis = self.genviz(vtopo)
+
+            # save federant information
+            for k in allocated.keys():
+                tbparams[k]['federant'] = {
+                        'name': [ { 'localname' : eid} ],
+                        'allocID' : tbparams[k]['allocID'],
+			'uri': tbparams[k]['uri'],
+                    }
+
+            self.state_lock.acquire()
+            self.state[eid]['vtopo'] = vtopo
+            self.state[eid]['vis'] = vis
+	    self.state[eid]['experimentdescription'] = \
+		    { 'topdldescription': top.to_dict() }
+            self.state[eid]['federant'] = \
+                    [ tbparams[tb]['federant'] for tb in tbparams.keys() \
+                        if tbparams[tb].has_key('federant') ]
+            if self.state_filename: 
+		self.write_state()
+            self.state_lock.release()
+        except service_error, e:
+            # If something goes wrong in the parse (usually an access error)
+            # clear the placeholder state.  From here on out the code delays
+            # exceptions.  Failing at this point returns a fault to the remote
+            # caller. 
+
+            self.state_lock.acquire()
+            del self.state[eid]
+            del self.state[expid]
+            if self.state_filename: self.write_state()
+            self.state_lock.release()
+            raise e
+
+
+        # Start the background swapper and return the starting state.  From
+        # here on out, the state will stick around a while.
+
+        # Let users touch the state
+        self.auth.set_attribute(fid, expid)
+        self.auth.set_attribute(expid, expid)
+        # Override fedids can manipulate state as well
+        for o in self.overrides:
+            self.auth.set_attribute(o, expid)
+
+        # Create a logger that logs to the experiment's state object as well as
+        # to the main log file.
+        alloc_log = logging.getLogger('fedd.experiment_control.%s' % eid)
+	alloc_collector = self.list_log(self.state[eid]['log'])
+        h = logging.StreamHandler(alloc_collector)
+        # XXX: there should be a global one of these rather than repeating the
+        # code.
+        h.setFormatter(logging.Formatter("%(asctime)s %(name)s %(message)s",
+                    '%d %b %y %H:%M:%S'))
+        alloc_log.addHandler(h)
+        
+	attrs = [ 
+		{
+		    'attribute': 'ssh_pubkey', 
+		    'value': '%s/%s/config/%s' % \
+			    (self.repo_url, expid, gw_pubkey_base)
+		},
+		{
+		    'attribute': 'ssh_secretkey', 
+		    'value': '%s/%s/config/%s' % \
+			    (self.repo_url, expid, gw_secretkey_base)
+		},
+		{
+		    'attribute': 'hosts', 
+		    'value': '%s/%s/config/hosts' % \
+			    (self.repo_url, expid)
+		},
+	    ]
+
+	# transit and disconnected testbeds may not have a connInfo entry.
+	# Fill in the blanks.
+	for t in allocated.keys():
+	    if not connInfo.has_key(t):
+		connInfo[t] = { }
+
+        # Start a thread to do the resource allocation
+        t  = Thread(target=self.allocate_resources,
+                args=(allocated, masters, eid, expid, tbparams, 
+                    top, topo, tmpdir, alloc_log, alloc_collector, attrs,
+		    connInfo, tbmap),
+                name=eid)
+        t.start()
+
+        rv = {
+                'experimentID': [
+                    {'localname' : eid }, { 'fedid': copy.copy(expid) }
+                ],
+                'experimentStatus': 'starting',
+            }
+
+        return rv
+    
+    def get_experiment_fedid(self, key):
+        """
+        find the fedid associated with the localname key in the state database.
+        """
+
+        rv = None
+        self.state_lock.acquire()
+        if self.state.has_key(key):
+            if isinstance(self.state[key], dict):
+                try:
+                    kl = [ f['fedid'] for f in \
+                            self.state[key]['experimentID']\
+                                if f.has_key('fedid') ]
+                except KeyError:
+                    self.state_lock.release()
+                    raise service_error(service_error.internal, 
+                            "No fedid for experiment %s when getting "+\
+                                    "fedid(!?)" % key)
+                if len(kl) == 1:
+                    rv = kl[0]
+                else:
+                    self.state_lock.release()
+                    raise service_error(service_error.internal, 
+                            "multiple fedids for experiment %s when " +\
+                                    "getting fedid(!?)" % key)
+            else:
+                self.state_lock.release()
+                raise service_error(service_error.internal, 
+                        "Unexpected state for %s" % key)
+        self.state_lock.release()
+        return rv
+
+    def check_experiment_access(self, fid, key):
+        """
+        Confirm that the fid has access to the experiment.  Though a request
+        may be made in terms of a local name, the access attribute is always
+        the experiment's fedid.
+        """
+        if not isinstance(key, fedid):
+            key = self.get_experiment_fedid(key)
+
+        if self.auth.check_attribute(fid, key):
+            return True
+        else:
+            raise service_error(service_error.access, "Access Denied")
+
+
+    def get_handler(self, path, fid):
+	self.log.info("Get handler %s %s" % (path, fid))
+	if self.auth.check_attribute(fid, path):
+	    return ("%s/%s" % (self.repodir, path), "application/binary")
+	else:
+	    return (None, None)
+
+    def get_vtopo(self, req, fid):
+        """
+        Return the stored virtual topology for this experiment
+        """
+        rv = None
+        state = None
+
+        req = req.get('VtopoRequestBody', None)
+        if not req:
+            raise service_error(service_error.req,
+                    "Bad request format (no VtopoRequestBody)")
+        exp = req.get('experiment', None)
+        if exp:
+            if exp.has_key('fedid'):
+                key = exp['fedid']
+                keytype = "fedid"
+            elif exp.has_key('localname'):
+                key = exp['localname']
+                keytype = "localname"
+            else:
+                raise service_error(service_error.req, "Unknown lookup type")
+        else:
+            raise service_error(service_error.req, "No request?")
+
+        self.check_experiment_access(fid, key)
+
+        self.state_lock.acquire()
+        if self.state.has_key(key):
+            if self.state[key].has_key('vtopo'):
+                rv = { 'experiment' : {keytype: key },\
+                        'vtopo': self.state[key]['vtopo'],\
+                    }
+            else:
+                state = self.state[key]['experimentStatus']
+        self.state_lock.release()
+
+        if rv: return rv
+        else: 
+            if state:
+                raise service_error(service_error.partial, 
+                        "Not ready: %s" % state)
+            else:
+                raise service_error(service_error.req, "No such experiment")
+
+    def get_vis(self, req, fid):
+        """
+        Return the stored visualization for this experiment
+        """
+        rv = None
+        state = None
+
+        req = req.get('VisRequestBody', None)
+        if not req:
+            raise service_error(service_error.req,
+                    "Bad request format (no VisRequestBody)")
+        exp = req.get('experiment', None)
+        if exp:
+            if exp.has_key('fedid'):
+                key = exp['fedid']
+                keytype = "fedid"
+            elif exp.has_key('localname'):
+                key = exp['localname']
+                keytype = "localname"
+            else:
+                raise service_error(service_error.req, "Unknown lookup type")
+        else:
+            raise service_error(service_error.req, "No request?")
+
+        self.check_experiment_access(fid, key)
+
+        self.state_lock.acquire()
+        if self.state.has_key(key):
+            if self.state[key].has_key('vis'):
+                rv =  { 'experiment' : {keytype: key },\
+                        'vis': self.state[key]['vis'],\
+                        }
+            else:
+                state = self.state[key]['experimentStatus']
+        self.state_lock.release()
+
+        if rv: return rv
+        else:
+            if state:
+                raise service_error(service_error.partial, 
+                        "Not ready: %s" % state)
+            else:
+                raise service_error(service_error.req, "No such experiment")
+
+    def clean_info_response(self, rv):
+        """
+        Remove the information in the experiment's state object that is not in
+        the info response.
+        """
+        # Remove the owner info (should always be there, but...)
+        if rv.has_key('owner'): del rv['owner']
+
+        # Convert the log into the allocationLog parameter and remove the
+        # log entry (with defensive programming)
+        if rv.has_key('log'):
+            rv['allocationLog'] = "".join(rv['log'])
+            del rv['log']
+        else:
+            rv['allocationLog'] = ""
+
+        if rv['experimentStatus'] != 'active':
+            if rv.has_key('federant'): del rv['federant']
+        else:
+            # remove the allocationID and uri info from each federant
+            for f in rv.get('federant', []):
+                if f.has_key('allocID'): del f['allocID']
+		if f.has_key('uri'): del f['uri']
+
+        return rv
+
+    def get_info(self, req, fid):
+        """
+        Return all the stored info about this experiment
+        """
+        rv = None
+
+        req = req.get('InfoRequestBody', None)
+        if not req:
+            raise service_error(service_error.req,
+                    "Bad request format (no InfoRequestBody)")
+        exp = req.get('experiment', None)
+        if exp:
+            if exp.has_key('fedid'):
+                key = exp['fedid']
+                keytype = "fedid"
+            elif exp.has_key('localname'):
+                key = exp['localname']
+                keytype = "localname"
+            else:
+                raise service_error(service_error.req, "Unknown lookup type")
+        else:
+            raise service_error(service_error.req, "No request?")
+
+        self.check_experiment_access(fid, key)
+
+        # The state may be massaged by the service function that called
+        # get_info (e.g., encoded for XMLRPC transport) so send a copy of the
+        # state.
+        self.state_lock.acquire()
+        if self.state.has_key(key):
+            rv = copy.deepcopy(self.state[key])
+        self.state_lock.release()
+
+        if rv:
+            return self.clean_info_response(rv)
+        else:
+            raise service_error(service_error.req, "No such experiment")
+
+    def get_multi_info(self, req, fid):
+        """
+        Return all the stored info that this fedid can access
+        """
+        rv = { 'info': [ ] }
+
+        self.state_lock.acquire()
+        for key in [ k for k in self.state.keys() if isinstance(k, fedid)]:
+	    try:
+		self.check_experiment_access(fid, key)
+	    except service_error, e:
+		if e.code == service_error.access:
+		    continue
+		else:
+		    self.state_lock.release()
+		    raise e
+
+            if self.state.has_key(key):
+                e = copy.deepcopy(self.state[key])
+                e = self.clean_info_response(e)
+                rv['info'].append(e)
+        self.state_lock.release()
+        return rv
+
+    def remove_dirs(self, dir):
+	"""
+	Remove the directory tree and all files rooted at dir.  Log any errors,
+	but continue.
+	"""
+	self.log.debug("[removedirs]: removing %s" % dir)
+	try:
+	    for path, dirs, files in os.walk(dir, topdown=False):
+		for f in files:
+		    os.remove(os.path.join(path, f))
+		for d in dirs:
+		    os.rmdir(os.path.join(path, d))
+	    os.rmdir(dir)
+	except EnvironmentError, e:
+	    self.log.error("Error deleting directory tree in %s" % e);
+
+    def terminate_experiment(self, req, fid):
+        """
+        Swap this experiment out on the federants and delete the shared
+        information
+        """
+        tbparams = { }
+        req = req.get('TerminateRequestBody', None)
+        if not req:
+            raise service_error(service_error.req,
+                    "Bad request format (no TerminateRequestBody)")
+        force = req.get('force', False)
+        exp = req.get('experiment', None)
+        if exp:
+            if exp.has_key('fedid'):
+                key = exp['fedid']
+                keytype = "fedid"
+            elif exp.has_key('localname'):
+                key = exp['localname']
+                keytype = "localname"
+            else:
+                raise service_error(service_error.req, "Unknown lookup type")
+        else:
+            raise service_error(service_error.req, "No request?")
+
+        self.check_experiment_access(fid, key)
+
+        dealloc_list = [ ]
+
+
+        # Create a logger that logs to the dealloc_list as well as to the main
+        # log file.
+        dealloc_log = logging.getLogger('fedd.experiment_control.%s' % key)
+        h = logging.StreamHandler(self.list_log(dealloc_list))
+        # XXX: there should be a global one of these rather than repeating the
+        # code.
+        h.setFormatter(logging.Formatter("%(asctime)s %(name)s %(message)s",
+                    '%d %b %y %H:%M:%S'))
+        dealloc_log.addHandler(h)
+
+        self.state_lock.acquire()
+        fed_exp = self.state.get(key, None)
+	repo = None
+
+        if fed_exp:
+            # This branch of the conditional holds the lock to generate a
+            # consistent temporary tbparams variable to deallocate experiments.
+            # It releases the lock to do the deallocations and reacquires it to
+            # remove the experiment state when the termination is complete.
+
+            # First make sure that the experiment creation is complete.
+            status = fed_exp.get('experimentStatus', None)
+
+            if status:
+                if status in ('starting', 'terminating'):
+                    if not force:
+                        self.state_lock.release()
+                        raise service_error(service_error.partial, 
+                                'Experiment still being created or destroyed')
+                    else:
+                        self.log.warning('Experiment in %s state ' % status + \
+                                'being terminated by force.')
+            else:
+                # No status??? trouble
+                self.state_lock.release()
+                raise service_error(service_error.internal,
+                        "Experiment has no status!?")
+
+            ids = []
+            #  experimentID is a list of dicts that are self-describing
+            #  identifiers.  This finds all the fedids and localnames - the
+            #  keys of self.state - and puts them into ids.
+            for id in fed_exp.get('experimentID', []):
+                if id.has_key('fedid'): 
+		    ids.append(id['fedid'])
+		    repo = "%s" % id['fedid']
+                if id.has_key('localname'): ids.append(id['localname'])
+
+	    # Collect the allocation/segment ids into a dict keyed by the fedid
+	    # of the allocation (or a monotonically increasing integer) that
+	    # contains a tuple of uri, aid (which is a dict...)
+	    for i, fed in enumerate(fed_exp.get('federant', [])):
+                try:
+		    uri = fed['uri']
+		    aid = fed['allocID']
+		    k = fed['allocID'].get('fedid', i)
+                except KeyError, e:
+                    continue
+                tbparams[k] = (uri, aid)
+            fed_exp['experimentStatus'] = 'terminating'
+            if self.state_filename: self.write_state()
+            self.state_lock.release()
+
+            # Stop everyone.  NB, wait_for_all waits until a thread starts and
+            # then completes, so we can't wait if nothing starts.  So, no
+            # tbparams, no start.
+            if len(tbparams) > 0:
+                thread_pool = self.thread_pool(self.nthreads)
+                for k in tbparams.keys():
+                    # Create and start a thread to stop the segment
+                    thread_pool.wait_for_slot()
+		    uri, aid = tbparams[k]
+                    t  = self.pooled_thread(\
+                            target=self.terminate_segment(log=dealloc_log,
+				testbed=uri,
+				cert_file=self.cert_file, 
+				cert_pwd=self.cert_pwd,
+				trusted_certs=self.trusted_certs,
+				caller=self.call_TerminateSegment),
+                            args=(uri, aid), name=k,
+                            pdata=thread_pool, trace_file=self.trace_file)
+                    t.start()
+                # Wait for completions
+                thread_pool.wait_for_all_done()
+
+            # release the allocations (failed experiments have done this
+            # already, and starting experiments may be in odd states, so we
+            # ignore errors releasing those allocations
+            try: 
+                for k in tbparams.keys():
+		    # This releases access by uri
+		    uri, aid = tbparams[k]
+                    self.release_access(None, aid, uri=uri)
+            except service_error, e:
+                if status != 'failed' and not force:
+                    raise e
+
+            # Remove the terminated experiment
+            self.state_lock.acquire()
+            for id in ids:
+                if self.state.has_key(id): del self.state[id]
+
+            if self.state_filename: self.write_state()
+            self.state_lock.release()
+
+	    # Delete any synch points associated with this experiment.  All
+	    # synch points begin with the fedid of the experiment.
+	    fedid_keys = set(["fedid:%s" % f for f in ids \
+		    if isinstance(f, fedid)])
+	    for k in self.synch_store.all_keys():
+		try:
+		    if len(k) > 45 and k[0:46] in fedid_keys:
+			self.synch_store.del_value(k)
+		except synch_store.BadDeletionError:
+		    pass
+	    self.write_store()
+
+	    # Remove software and other cached stuff from the filesystem.
+	    if repo:
+		self.remove_dirs("%s/%s" % (self.repodir, repo))
+		
+            return { 
+                    'experiment': exp , 
+                    'deallocationLog': "".join(dealloc_list),
+                    }
+        else:
+            # Don't forget to release the lock
+            self.state_lock.release()
+            raise service_error(service_error.req, "No saved state")
+
+
+    def GetValue(self, req, fid):
+	"""
+	Get a value from the synchronized store
+	"""
+        req = req.get('GetValueRequestBody', None)
+        if not req:
+            raise service_error(service_error.req,
+                    "Bad request format (no GetValueRequestBody)")
+	
+	name = req['name']
+	wait = req['wait']
+	rv = { 'name': name }
+
+        if self.auth.check_attribute(fid, name):
+	    self.log.debug("[GetValue] asking for %s " % name)
+	    try:
+		v = self.synch_store.get_value(name, wait)
+	    except synch_store.RevokedKeyError:
+		# No more synch on this key
+		raise service_error(service_error.federant, 
+			"Synch key %s revoked" % name)
+	    if v is not None:
+		rv['value'] = v
+	    self.log.debug("[GetValue] got %s from %s" % (v, name))
+	    return rv
+        else:
+            raise service_error(service_error.access, "Access Denied")
+	
+
+    def SetValue(self, req, fid):
+	"""
+	Set a value in the synchronized store
+	"""
+        req = req.get('SetValueRequestBody', None)
+        if not req:
+            raise service_error(service_error.req,
+                    "Bad request format (no SetValueRequestBody)")
+	
+	name = req['name']
+	v = req['value']
+
+        if self.auth.check_attribute(fid, name):
+	    try:
+		self.synch_store.set_value(name, v)
+		self.write_store()
+		self.log.debug("[SetValue] set %s to %s" % (name, v))
+	    except synch_store.CollisionError:
+		# Translate into a service_error
+		raise service_error(service_error.req,
+			"Value already set: %s" %name)
+	    except synch_store.RevokedKeyError:
+		# No more synch on this key
+		raise service_error(service_error.federant, 
+			"Synch key %s revoked" % name)
+	    return { 'name': name, 'value': v }
+        else:
+            raise service_error(service_error.access, "Access Denied")
Index: fedd/Makefile/federation/experiment_partition.py
===================================================================
--- fedd/Makefile/federation/experiment_partition.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/experiment_partition.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,673 @@
+#!/usr/local/bin/python
+
+import os,sys
+
+import re
+import random
+import string
+import subprocess
+import tempfile
+import copy
+import pickle
+import logging
+import signal
+import time
+
+import traceback
+# For parsing visualization output and splitter output
+import xml.parsers.expat
+
+from threading import Lock, Thread, Condition
+from subprocess import call, Popen, PIPE
+
+from urlparse import urlparse
+from urllib2 import urlopen
+
+from util import *
+from fedid import fedid, generate_fedid
+from remote_service import xmlrpc_handler, soap_handler, service_caller
+from service_error import service_error
+from synch_store import synch_store
+
+import topdl
+import list_log
+from ip_allocator import ip_allocator
+from ip_addr import ip_addr
+
+
+class nullHandler(logging.Handler):
+    def emit(self, record): pass
+
+fl = logging.getLogger("fedd.experiment_control")
+fl.addHandler(nullHandler())
+
+class experiment_partition: 
+    def __init__(self, auth=None, store_url=None, tbmap=None, muxmax=2, 
+	    direct_transit=None):
+        """
+        Intialize the various attributes
+        """
+
+	self.log = logging.getLogger("fedd.experiment_control." + \
+		"experiment_paritition")
+	self.auth = auth
+	self.store_url = store_url
+	self.tbmap = tbmap
+	self.direct_transit = direct_transit or [ ]
+	self.muxmax = muxmax
+
+
+    def new_portal_node(self, st, dt, tbparams, masters, eid, myname, desthost,
+	    portal_type, iface_desc=(), conn_type="ssh", conn_attrs=[], 
+	    expid=None):
+	"""
+	Return a new internet portal node and a dict with the connectionInfo to
+	be attached.
+	"""
+	seer_master = None
+	for m in masters.values():
+	    for s in m:
+		if s.name == 'SEER':
+		    seer_master = m
+		    break
+	    if seer_master: break
+
+	if seer_master:
+	    mdomain = tbparams[seer_master].get('domain', '.example.com')
+	    mproject = tbparams[seer_master].get('project', 'project')
+	    muser = tbparams[seer_master].get('user', 'root')
+	    smbshare = tbparams[seer_master].get('smbshare', 'USERS')
+	else:
+	    mdomain = '.example.com'
+	    mproject = 'project'
+	    muser = 'root'
+	    smbshare = 'USERS'
+
+	dproject = tbparams[dt].get('project', 'project')
+	ddomain = tbparams[dt].get('domain', '.example.com')
+
+	if (st in masters and dt not in masters) or \
+		( st not in masters and dt in masters ):
+	    active = ("%s" % (st in masters))
+	else:
+	    active = ("%s" % (st > dt))
+
+	ifaces = [ ]
+	for sub, attrs in iface_desc:
+	    inf = topdl.Interface(
+		    name="inf%03d" % len(ifaces),
+		    substrate=sub,
+		    attribute=[
+			topdl.Attribute(
+			    attribute=n,
+			    value = v)
+			for n, v in attrs
+			]
+		    )
+	    ifaces.append(inf)
+	if conn_type == "ssh":
+	    try:
+		aid = tbparams[st]['allocID']['fedid']
+	    except:
+		self.log.debug("[new_portal_node] Can't get alloc id for %s?" \
+			% st)
+		aid = None
+	    info = {
+		    "type" : conn_type, 
+		    "portal": myname,
+		    'fedAttr': [ 
+			    { 'attribute': 'masterdomain', 'value': mdomain},
+			    { 'attribute': 'masterexperiment', 'value': 
+				"%s/%s" % (mproject, eid)},
+			    { 'attribute': 'active', 'value': active},
+			    # Move to SMB service description
+			    { 'attribute': 'masteruser', 'value': muser},
+			    { 'attribute': 'smbshare', 'value': smbshare},
+			],
+		    'parameter': [
+			{
+			    'name': 'peer',
+			    'key': 'fedid:%s/%s' % (expid, myname),
+			    'store': self.store_url,
+			    'type': 'output',
+			},
+			{
+			    'name': 'ssh_port',
+			    'key': 'fedid:%s/%s-port' % (expid, myname),
+			    'store': self.store_url,
+			    'type': 'output',
+			},
+			{
+			    'name': 'peer',
+			    'key': 'fedid:%s/%s' % (expid, desthost),
+			    'store': self.store_url,
+			    'type': 'input',
+			},
+			{
+			    'name': 'ssh_port',
+			    'key': 'fedid:%s/%s-port' % (expid, desthost),
+			    'store': self.store_url,
+			    'type': 'input',
+			},
+			]
+		    }
+	    # Give this allocation the rights to access the key of the
+	    # peers
+	    if aid:
+		for h in (myname, desthost):
+		    self.auth.set_attribute(aid, 'fedid:%s/%s' % (expid, h))
+		    self.auth.set_attribute(aid, 'fedid:%s/%s-port' % \
+			    (expid, h))
+	    else:
+		self.log.error("No aid for %s in new_portal_node" % st)
+	else:
+	    info = None
+
+	return (topdl.Computer(
+		name=myname,
+		attribute=[ 
+		    topdl.Attribute(attribute=n,value=v)
+			for n, v in (\
+			    ('portal', 'true'),
+			    ('portal_type', portal_type), 
+			    ('destination_testbed', dt),
+			)
+		    ],
+		interface=ifaces,
+		), info)
+
+    def new_portal_substrate(self, st, dt, eid, tbparams, expid):
+	ddomain = tbparams[dt].get('domain', ".example.com")
+	dproject = tbparams[dt].get('project', 'project')
+	tsubstrate = \
+		topdl.Substrate(name='%s-%s' % (st, dt),
+			attribute= [
+			    topdl.Attribute(
+				attribute='portal',
+				value='true')
+			    ]
+			)
+	segment_element = topdl.Segment(
+		id= tbparams[dt]['allocID'],
+		type='emulab',
+		uri = self.tbmap.get(testbed_base(dt), None),
+		interface=[ 
+		    topdl.Interface(
+			substrate=tsubstrate.name),
+		    ],
+		attribute = [
+		    topdl.Attribute(attribute=n, value=v)
+			for n, v in (\
+			    ('domain', ddomain),
+			    ('experiment', "%s/%s" % \
+				    (dproject, eid)),)
+		    ],
+		)
+
+	return (tsubstrate, segment_element)
+
+    def new_direct_topo(self, idx, sub, topo, tbs, tbparams, connInfo, expid,
+	    tb_name):
+	if sub.capacity is None:
+	    raise service_error(service_error.internal,
+		    "Cannot direct split substrate w/o capacity")
+	segs = [ ]
+	name = join_testbed(tb_name, "%d" % idx)
+	substr = topdl.Substrate(name=name, 
+		capacity=sub.capacity.clone(),
+		attribute=[ topdl.Attribute(attribute=n, value=v)
+		    for n, v, in (\
+			    ('vlan', 'unassigned%d' % idx),)])
+	store_key = 'fedid:%s/vlan%d' % (expid, idx)
+	for tb in tbs.keys():
+	    seg = topdl.Segment(
+		    id = tbparams[tb]['allocID'],
+		    type='emulab',
+		    uri = self.tbmap.get(testbed_base(tb), None),
+		    interface=[ 
+			topdl.Interface(
+			    substrate=substr.name),
+			],
+		    attribute=[ topdl.Attribute(
+			attribute='%s_endpoint' % tb_name, 
+			value=tbparams[tb][tb_name]),
+			]
+		    )
+	    vlan_key = "%s_vlans" % tb_name
+	    if vlan_key in tbparams[tb]:
+		seg.set_attribute(vlan_key, tbparams[tb][vlan_key])
+	    segs.append(seg)
+
+	    # Give this allocation the rights to access the key of the
+	    # vlan_id
+	    try:
+		aid = tbparams[tb]['allocID']['fedid']
+		self.auth.set_attribute(aid, store_key)
+	    except:
+		self.log.debug("[new_direct_topo] Can't get alloc id for %s?"\
+			% tb)
+
+	connInfo[name] = [ { 
+	    'type': 'transit',
+	    'parameter': [ { 
+		'name': 'vlan_id',
+		'key': store_key,
+		'store': self.store_url,
+		'type': 'output'
+		} ]
+	    } ]
+
+	topo[name] = \
+		topdl.Topology(substrates=[substr], elements=segs,
+			attribute=[
+			    topdl.Attribute(attribute="transit", value='true'),
+			    topdl.Attribute(attribute="dynamic", value='true'),
+			    topdl.Attribute(attribute="testbed", value=tb_name),
+			    topdl.Attribute(attribute="store_keys", 
+				value=store_key),
+			    ]
+			)
+
+    def create_direct_substrate(self, sub, topo, tbs, tbparams, masters, eid,
+	    connInfo, expid=None, tb_name=None):
+	"""
+	Create connection information that tells which nodes are to be
+	connected to direct transits, and create an additional topology with
+	just the interconnected segments and a substrate.
+	"""
+
+	def get_substrate_from_topo(name, t):
+	    for s in t.substrates:
+		if s.name == name: return s
+	    else: return None
+
+
+	seer_master = None
+	for m in masters.values():
+	    for s in m:
+		if s.name == 'SEER':
+		    seer_master = m
+		    break
+	    if seer_master: break
+
+	if seer_master:
+	    mdomain = tbparams[seer_master].get('domain', '.example.com')
+	    mproject = tbparams[seer_master].get('project', 'project')
+	else:
+	    mdomain = '.example.com'
+	    mproject = 'project'
+
+	# dn is the number of previously created direct nets on this direct
+	# testbed.  This routine creates a net numbered by dn
+	dn = len([x for x in topo.keys() if x.startswith(tb_name)])
+
+	# Set the attributes in the copies that will allow setup of direct
+	# connections.
+	for tb in tbs.keys():
+	    s = get_substrate_from_topo(sub.name, topo[tb])
+	    if s:
+		if not connInfo.has_key(tb):
+		    connInfo[tb] = [ ]
+
+		try:
+		    aid = tbparams[tb]['allocID']['fedid']
+		except:
+		    self.log.debug("[create_direct_substrate] " + 
+			    "Can't get alloc id for %s?" %tb)
+		    aid = None
+
+		# This may need another look, but only a service gateway will
+		# look at the active parameter, and these are only inserted to
+		# connect to a master.
+		active = "%s" % ( tb in masters)
+		info = {
+			'type': 'transit',
+			'member': [ {
+			    'element': i.element.name, 
+			    'interface': i.name 
+			    } for i in s.interfaces \
+				    if isinstance(i.element, topdl.Computer) ],
+			'fedAttr': [ 
+			    { 'attribute': 'masterdomain', 'value': mdomain},
+			    { 'attribute': 'masterexperiment', 'value': 
+				"%s/%s" % (mproject, eid)},
+			    { 'attribute': 'active', 'value': active},
+			    ],
+			'parameter': [ {
+			    'name': 'vlan_id',
+			    'key': 'fedid:%s/vlan%d' % (expid, dn),
+			    'store': self.store_url,
+			    'type': 'input',
+			    } ]
+			}
+		if tbs.has_key(tb):
+		    info['peer'] = tbs[tb]
+		connInfo[tb].append(info)
+
+		# Give this allocation the rights to access the key of the
+		# vlan_id
+		if aid:
+		    self.auth.set_attribute(aid, 
+			    'fedid:%s/vlan%d' % (expid, dn))
+	    else:
+		raise service_error(service_error.internal,
+			"No substrate %s in testbed %s" % (sub.name, tb))
+
+	self.new_direct_topo(dn, sub, topo, tbs, tbparams, connInfo, expid,
+		tb_name)
+
+    def insert_internet_portals(self, sub, topo, tbs, tbparams, masters, eid,
+	    segment_substrate, portals, connInfo, expid):
+	# More than one testbed is on this substrate.  Insert
+	# some portals into the subtopologies.  st == source testbed,
+	# dt == destination testbed.
+	for st in tbs.keys():
+	    if not segment_substrate.has_key(st):
+		segment_substrate[st] = { }
+	    if not portals.has_key(st): 
+		portals[st] = { }
+	    if not connInfo.has_key(st):
+		connInfo[st] = [ ]
+	    for dt in [ t for t in tbs.keys() if t != st]:
+		sproject = tbparams[st].get('project', 'project')
+		dproject = tbparams[dt].get('project', 'project')
+		sdomain = tbparams[st].get('domain', ".example.com")
+		ddomain = tbparams[dt].get('domain', ".example.com")
+		aid = tbparams[dt]['allocID']['fedid']
+
+		seer_master = None
+
+		for m in masters.values():
+		    for s in m:
+			if s.name == 'SEER':
+			    seer_master = m
+			    break
+		    if seer_master: break
+
+		if seer_master:
+		    mdomain = tbparams[seer_master].get('domain', '.example.com')
+		    mproject = tbparams[seer_master].get('project', 'project')
+		    muser = tbparams[seer_master].get('user', 'root')
+		    smbshare = tbparams[seer_master].get('smbshare', 'USERS')
+		else:
+		    mdomain = '.example.com'
+		    mproject = 'project'
+		    muser = 'root'
+		    smbshare = 'USERS'
+
+		if (st in masters  and dt not in masters) or \
+			(st not in masters and dt in masters):
+		    active = ("%s" % (st in masters))
+		else:
+		    active = ("%s" %(st > dt))
+
+		if not segment_substrate[st].has_key(dt):
+		    # Put a substrate and a segment for the connected
+		    # testbed in there.
+		    tsubstrate, segment_element = \
+			    self.new_portal_substrate(st, dt, eid, tbparams,
+				    expid)
+		    segment_substrate[st][dt] = tsubstrate
+		    topo[st].substrates.append(tsubstrate)
+		    topo[st].elements.append(segment_element)
+
+		new_portal = False
+		if testbed_suffix(dt): dname = "-".join(split_testbed(dt))
+		else: dname = dt
+
+		if testbed_suffix(st): sname = "-".join(split_testbed(st))
+		else: sname = st
+
+		if portals[st].has_key(dt):
+		    # There's a portal set up to go to this destination.
+		    # See if there's room to multiplex this connection on
+		    # it.  If so, add an interface to the portal; if not,
+		    # set up to add a portal below.
+		    # [This little festival of braces is just a pop of the
+		    # last element in the list of portals between st and
+		    # dt.]
+		    portal = portals[st][dt][-1]
+		    mux = len([ i for i in portal.interface \
+			    if not i.get_attribute('portal')])
+		    if mux == self.muxmax:
+			new_portal = True
+			portal_type = "experiment"
+			myname = "%stunnel%d" % (dname.lower(),
+				len(portals[st][dt]))
+			desthost = "%stunnel%d" % (sname.lower(), 
+				len(portals[st][dt]))
+		    else:
+			new_i = topdl.Interface(
+				substrate=sub.name,
+				attribute=[ 
+				    topdl.Attribute(
+					attribute='ip4_address', 
+					value=tbs[dt]
+				    )
+				])
+			portal.interface.append(new_i)
+		else:
+		    # First connection to this testbed, make an empty list
+		    # and set up to add the new portal below
+		    new_portal = True
+		    portals[st][dt] = [ ]
+		    myname = "%stunnel%d" % (dname.lower(),
+			    len(portals[st][dt]))
+		    desthost = "%stunnel%d" % (sname.lower(),
+			    len(portals[st][dt]))
+
+		    if dt in masters or st in masters: portal_type = "both"
+		    else: portal_type = "experiment"
+
+		if new_portal:
+		    infs = (
+			    (segment_substrate[st][dt].name, 
+				(('portal', 'true'),)),
+			    (sub.name, 
+				(('ip4_address', tbs[dt]),))
+			)
+		    portal, info  =  self.new_portal_node(st, dt, tbparams, 
+			    masters, eid, myname, desthost, portal_type,
+			    infs, conn_type="ssh", conn_attrs=[], expid=expid)
+
+		    topo[st].elements.append(portal)
+		    portals[st][dt].append(portal)
+		    connInfo[st].append(info)
+
+    def add_control_portal(self, st, dt, masters, eid, topo, tbparams, 
+	    connInfo, expid):
+	# Add to the master testbed
+	tsubstrate, segment_element = \
+		self.new_portal_substrate(st, dt, eid, tbparams, expid)
+	myname = "%stunnel" % dt
+	desthost = "%stunnel" % st
+
+	portal, info = self.new_portal_node(st, dt, tbparams, masters,
+		eid, myname, desthost, "control", 
+		((tsubstrate.name,(('portal','true'),)),), conn_type="ssh",
+		conn_attrs=[], expid=expid)
+
+	topo[st].substrates.append(tsubstrate)
+	topo[st].elements.append(segment_element)
+	topo[st].elements.append(portal)
+	if not connInfo.has_key(st):
+	    connInfo[st] = [ ]
+	connInfo[st].append(info)
+
+    def new_direct_portal(self, st, dt, masters, eid, myip, dip, idx, 
+	    substrate, tbparams, expid, tb_name):
+	# Add to the master testbed
+	if testbed_suffix(dt): myname = "%stunnel" % "-".join(split_testbed(dt))
+	else: myname = "%stunnel" % dt
+
+	desthost = "%s" % ip_addr(dip)
+
+	portal, info = self.new_portal_node(st, dt, tbparams, masters,
+		eid, myname, desthost, "control", 
+		((substrate.name,(
+		    ('portal','true'),
+		    ('ip4_address', "%s" % ip_addr(myip)),)),),
+		conn_type="transit", conn_attrs=[], expid=expid)
+
+	return portal
+
+    def add_portals(self, top, topo, eid, masters, tbparams, ip_allocator, 
+	    connInfo, expid):
+	"""
+	For each substrate in the main topology, find those that
+	have nodes on more than one testbed.  Insert portal nodes
+	into the copies of those substrates on the sub topologies.
+	"""
+	segment_substrate = { }
+	portals = { }
+	for s in top.substrates:
+	    # tbs will contain an ip address on this subsrate that is in
+	    # each testbed.
+	    tbs = { }
+	    for i in s.interfaces:
+		e = i.element
+		tb = e.get_attribute('testbed')
+		if tb and tb not in tbs:
+		    tbs[tb]= i.get_attribute('ip4_address')
+	    if len(tbs) < 2:
+		continue
+
+	    base_tbs = set([testbed_base(t) for t in tbs])
+
+	    # DRAGON will not create multi-site vlans yet, so we don't do multi
+	    # connection direct transits yet.
+	    if len(tbs) == 2 :
+		# NB: the else if on the for loop - if none of the direct
+		# transits is applicable, use the internet.
+		for d in self.direct_transit:
+		    if all([tbparams[x].has_key(d) for x in tbs]):
+			self.create_direct_substrate(s, topo, tbs, tbparams, 
+				masters, eid, connInfo, expid, d)
+			break
+		else:
+		    self.insert_internet_portals(s, topo, tbs, tbparams,
+			    masters, eid, segment_substrate, portals,
+			    connInfo, expid)
+	    else:
+		self.insert_internet_portals(s, topo, tbs, tbparams, masters,
+			eid, segment_substrate, portals, connInfo, expid)
+
+
+	# Make sure that all the service importers have a control portal back
+	# to the master for each service.
+	for mtb in [ t for t in tbparams if t in masters ]:
+	    importers = set([])
+	    for m in masters[mtb]:
+		importers |= set(m.importers)
+	    if mtb in importers:
+		importers.discard(mtb)
+	    for tb in importers:
+		if tb not in topo:
+		    self.log.error("Importer not in experiment: %s" % tb)
+		    continue
+		if len([e for e in topo[tb].elements \
+			if isinstance(e, topdl.Computer) and \
+			e.get_attribute('destination_testbed') == mtb and \
+			e.get_attribute('portal') and \
+			e.get_attribute('portal_type') == 'both']) == 0:
+
+		    for tb_name in self.direct_transit:
+
+			if tbparams[mtb].has_key(tb_name) \
+				and tbparams[tb].has_key(tb_name):
+
+			    idx = len([x for x in topo.keys() \
+				    if x.startswith(tb_name)])
+			    dip, leng = ip_allocator.allocate(4)
+			    dip += 1
+			    mip = dip+1
+			    csub = topdl.Substrate(
+				    name="%s-control-%s" % (tb_name, tb),
+				    capacity=topdl.Capacity(100000.0, 'max'),
+				    attribute=[
+					topdl.Attribute(
+					    attribute='portal',
+					    value='true'
+					    )
+					]
+				    )
+			    seg = topdl.Segment(
+				    id= tbparams[mtb]['allocID'],
+				    type='emulab',
+				    uri = self.tbmap.get(testbed_base(mtb),
+					None),
+				    interface=[ 
+					topdl.Interface(
+					    substrate=csub.name),
+					],
+				    attribute = [
+					topdl.Attribute(attribute=n, value=v)
+					    for n, v in (\
+						('domain', 
+						    tbparams[mtb].get('domain',
+							".example.com")),
+						('experiment', "%s/%s" % \
+							(tbparams[mtb].get(
+							    'project', 
+							    'project'), 
+							    eid)),)
+					],
+				    )
+			    portal = self.new_direct_portal(tb, mtb,
+				    masters, eid, dip, mip, idx, csub,
+				    tbparams, expid, tb_name)
+			    topo[tb].substrates.append(csub)
+			    topo[tb].elements.append(portal)
+			    topo[tb].elements.append(seg)
+
+			    mcsub = csub.clone()
+			    seg = topdl.Segment(
+				    id= tbparams[tb]['allocID'],
+				    type='emulab',
+				    uri = self.tbmap.get(testbed_base(tb),
+					None),
+				    interface=[ 
+					topdl.Interface(
+					    substrate=csub.name),
+					],
+				    attribute = [
+					topdl.Attribute(attribute=n, value=v)
+					    for n, v in (\
+						('domain', 
+						    tbparams[tb].get('domain',
+							".example.com")),
+						('experiment', "%s/%s" % \
+							(tbparams[tb].get('project', 
+							    'project'), 
+							    eid)),)
+					],
+				    )
+			    portal = self.new_direct_portal(mtb, tb, masters,
+				    eid, mip, dip, idx, mcsub, tbparams, expid,
+				    tb_name)
+			    topo[mtb].substrates.append(mcsub)
+			    topo[mtb].elements.append(portal)
+			    topo[mtb].elements.append(seg)
+			    for t in (mtb, tb):
+				topo[t].incorporate_elements()
+
+			    self.create_direct_substrate(csub, topo, 
+				    {tb: ip_addr(mip), mtb: ip_addr(dip)}, 
+				    tbparams, masters, eid, connInfo,
+				    expid, tb_name)
+			    break
+		    # This matches with the for tb_name in self.direct_transit
+		    else:
+			self.add_control_portal(mtb, tb, masters, eid, topo, 
+				tbparams, connInfo, expid)
+			self.add_control_portal(tb, mtb, masters, eid, topo, 
+				tbparams, connInfo, expid)
+
+	# Connect the portal nodes into the topologies and clear out
+	# substrates that are not in the topologies
+	for tb in tbparams:
+	    topo[tb].incorporate_elements()
+	    topo[tb].substrates = \
+		    [s for s in topo[tb].substrates \
+			if len(s.interfaces) >0]
+
Index: fedd/Makefile/federation/fedid.py
===================================================================
--- fedd/Makefile/federation/fedid.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/fedid.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,176 @@
+#!/usr/local/bin/python
+
+import os, sys
+import subprocess
+import tempfile
+import logging
+
+from M2Crypto import SSL, X509, EVP
+from pyasn1.codec.der import decoder
+
+# The version of M2Crypto on users is pretty old and doesn't have all the
+# features that are useful.  The legacy code is somewhat more brittle than the
+# main line, but will work.
+if "as_der" not in dir(EVP.PKey):
+    from asn1_raw import get_key_bits_from_file, get_key_bits_from_cert
+    legacy = True
+else:
+    legacy = False
+
+class fedid:
+    """
+    Wrapper around the federated ID from an X509 certificate. 
+    """
+    HASHSIZE=20
+    def __init__(self, bits=None, hexstr=None, cert=None, file=None):
+	if bits != None:
+	    self.set_bits(bits)
+	elif hexstr != None:
+	    self.set_hexstr(hexstr)
+	elif cert != None:
+	    self.set_cert(cert)
+	elif file != None:
+	    self.set_file(file)
+	else:
+	    self.buf = None
+
+    def __hash__(self):
+	return hash(self.buf)
+
+    def __eq__(self, other):
+	if isinstance(other, type(self)):
+	    return self.buf == other.buf
+	elif isinstance(other, type(str())):
+	    return self.buf == other;
+	else:
+	    return False
+
+    def __ne__(self, other): return not self.__eq__(other)
+
+    def __str__(self):
+	if self.buf != None:
+	    return str().join([ "%02x" % ord(x) for x in self.buf])
+	else: return ""
+
+    def __repr__(self):
+	return "fedid(hexstr='%s')" % self.__str__()
+
+    def pack_soap(self):
+	return self.buf
+
+    def pack_xmlrpc(self):
+	return self.buf
+
+    def digest_bits(self, bits):
+	"""Internal function.  Compute the fedid from bits and store in buf"""
+	d = EVP.MessageDigest('sha1')
+	d.update(bits)
+	self.buf = d.final()
+
+
+    def set_hexstr(self, hexstr):
+	h = hexstr.replace(':','')
+	self.buf= str().join([chr(int(h[i:i+2],16)) \
+		for i in range(0,2*fedid.HASHSIZE,2)])
+
+    def get_hexstr(self):
+	"""Return the hexstring representation of the fedid"""
+	return self.__str__()
+
+    def set_bits(self, bits):
+	"""Set the fedid to bits(a 160 bit buffer)"""
+	self.buf = bits
+
+    def get_bits(self):
+	"""Get the 160 bit buffer from the fedid"""
+	return self.buf
+
+    def set_file(self, file):
+	"""Get the fedid from a certificate file
+
+	Calculate the SHA1 hash over the bit string of the public key as
+	defined in RFC3280.
+	"""
+	self.buf = None
+	if legacy: self.digest_bits(get_key_bits_from_file(file))
+	else: self.set_cert(X509.load_cert(file))
+
+    def set_cert(self, cert):
+	"""Get the fedid from a certificate.
+
+	Calculate the SHA1 hash over the bit string of the public key as
+	defined in RFC3280.
+	"""
+
+	self.buf = None
+	if (cert != None):
+	    if legacy:
+		self.digest_bits(get_key_bits_from_cert(cert))
+	    else:
+		b = []
+		k = cert.get_pubkey()
+
+		# Getting the key was easy, but getting the bit string of the
+		# key requires a side trip through ASN.1
+		dec = decoder.decode(k.as_der())
+
+		# kv is a tuple of the bits in the key.  The loop below
+		# recombines these into bytes and then into a buffer for the
+		# SSL digest function.
+		kv =  dec[0].getComponentByPosition(1)
+		for i in range(0, len(kv), 8):
+		    v = 0
+		    for j in range(0, 8):
+			v = (v << 1) + kv[i+j]
+		    b.append(v)
+		# The comprehension turns b from a list of bytes into a buffer
+		# (string) of bytes
+		self.digest_bits(str().join([chr(x) for x in b]))
+
+def generate_fedid(subj, bits=2048, log=None, dir=None, trace=sys.stderr,
+	ssl_prog="/usr/bin/openssl"):
+    """
+    Create a new certificate and derive a fedid from it.
+
+    The fedid and the certificate are returned as a tuple.
+    """
+
+    keypath = None
+    certpath = None
+    try:
+	try:
+	    kd, keypath = tempfile.mkstemp(dir=dir, prefix="key",
+		    suffix=".pem")
+	    cd, certpath = tempfile.mkstemp(dir=dir, prefix="cert",
+		    suffix=".pem")
+
+	    cmd = [ssl_prog, "req", "-text", "-newkey", 
+		    "rsa:%d" % bits, "-keyout", keypath,  "-nodes", 
+		    "-subj", "/CN=%s" % subj, "-x509", "-days", "30", 
+		    "-out", certpath]
+
+	    if log:
+		log.debug("[generate_fedid] %s" % " ".join(cmd))
+
+	    if trace: call_out = trace
+	    else: 
+		call_out = open("/dev/null", "w")
+
+	    rv = subprocess.call(cmd, stdout=call_out, stderr=call_out)
+	    log.debug("rv = %d" % rv)
+	    if rv == 0:
+		cert = ""
+		for p in (certpath, keypath):
+		    f = open(p)
+		    for line in f:
+			cert += line
+		
+		fid = fedid(file=certpath)
+		return (fid, cert)
+	    else:
+		return (None, None)
+	except EnvironmentError, e:
+	    raise e
+    finally:
+	if keypath: os.remove(keypath)
+	if certpath: os.remove(certpath)
Index: fedd/Makefile/federation/geniapi_proxy.py
===================================================================
--- fedd/Makefile/federation/geniapi_proxy.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/geniapi_proxy.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,117 @@
+#!/usr/local/bin/python
+
+import os,sys
+
+from M2Crypto.SSL import SSLError
+from M2Crypto.m2xmlrpclib import SSL_Transport
+from service_error import service_error
+from xmlrpclib import ServerProxy, Fault, Error
+
+
+from protogeni_proxy import protogeni_proxy
+
+import topdl
+import list_log
+
+class geniapi_proxy(protogeni_proxy):
+    '''
+    This exports the same interface as the protogeni_proxy, but uses GENIAPI
+    calls when avaliable.  These calls are currently only available only for
+    the component manager/aggregate manager.  The keyword arguments of the
+    ProtoGENI interface are converted into XMLRPC positional parameters on
+    GENIAPI calls.  Tables to do that are established in the constructior.
+    '''
+    def __init__(self, log=None, keyfile=None, debug=False, 
+	    ch_url=None, sa_url=None, cm_url=None):
+	protogeni_proxy.__init__(self, log, keyfile, debug, ch_url, sa_url,
+		cm_url)
+
+	def get_key(k, d):
+	    '''
+	    Return a function that takes a dict and returns the value bound to
+	    key k or the default d
+	    '''
+	    return lambda x : x.get(k, d)
+
+	def user_filter(params):
+	    '''
+	    Take a dict assumed to hold a key 'users' that is a dict of a urn
+	    (keyed by 'urn' and a list of dicts describing ssh keys (keyed by
+	    'key'.  Construct a list of dicts with the same 'urn' and a 'keys'
+	    value holding only the 'key' value of the subdict.  This converts
+	    ProtoGENI users to GENIAPI users.
+	    '''
+
+	    # That conversion is a nested list comprehension.
+	    return [ { 
+		'urn': u.get('urn', ''), 
+		'keys': [ k.get('key', '') for k in u.get('keys', []) ] 
+		} for u in params.get('users', [])]
+
+	self.ProtoGENIError = protogeni_proxy.ProtoGENIError
+	# These ProtoGENI requests are mapped to the given GENIAPI request.
+	self.method_map = { 
+		'DeleteSlice': 'DeleteSliver',
+		'RenewSlice': 'RenewSliver',
+		}
+	# Conversion tables for changing the dict of keyword arguments into a
+	# list of arguments for GENIAPI.  The list of positional params is
+	# created by calling the functions in the dict below (keyed by the
+	# requested function) in order on the params dict, placing the results
+	# into the arguments list.  The list so constructed is the positional
+	# parameters in order.  Mostly this is just putting the right key in
+	# the right place, but the users choice is actually reformatted some.
+	# This dict is indexed by ProtoGENI method.
+	self.arg_map = {
+		'CreateSliver': [
+		    get_key('slice_urn', ''), 
+		    get_key('credentials', []), 
+		    get_key('rspec', ''), 
+		    user_filter ],
+		'SliverStatus': [
+		    get_key('slice_urn', ''),
+		    get_key('credentials','') ],
+		'DeleteSlice': [
+		    get_key('slice_urn', ''),
+		    get_key('credentials','') ],
+		'RenewSlice': [
+		    get_key('slice_urn', ''),
+		    get_key('credentials','') ],
+		}
+
+    def geniapi_call(self, url, method, params, context):
+	'''
+	Make an SSL authenticated (and encrypted) GENIAPI call on the given url.
+	'''
+	try:
+	    if self.log:
+		self.log.debug("Calling %s %s" % (url, method))
+	    transport = SSL_Transport(context)
+	    port = ServerProxy(url, transport=transport)
+	    remote_method = getattr(port, method, None)
+	    if remote_method is not None:
+		return remote_method(*params)
+	    else:
+		raise service_error(service_error.internal, 
+			"Bad method: %s" % method)
+	except Fault, f:
+	    raise service_error(f.faultCode, f.faultString)
+	except Error, e:
+	    raise service_error(service_error.protocol, 
+		    "Remote XMLRPC Fault: %s" % e)
+	except:
+	    raise self.ProtoGENIError(op=method, code=-1, 
+		    output="%s" % sys.exc_info()[1])
+
+
+    def component_manager_call(self, method, params, context):
+	'''
+	Convert the ProtoGENI call into a GENIAPI call using the translation
+	tables above.
+	'''
+	gmethod = self.method_map.get(method, method)
+	gparams = [f(params) for f in self.arg_map.get(method, [])]
+
+	return self.geniapi_call(self.cm_url, gmethod, gparams, context)
+
+
Index: fedd/Makefile/federation/ip_addr.py
===================================================================
--- fedd/Makefile/federation/ip_addr.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/ip_addr.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,74 @@
+#!/usr/local/bin/python
+
+class ip_addr:
+    def __init__(self, x):
+	if isinstance(x, ip_addr):
+	    self.ival = x.ival
+	elif isinstance(x, int):
+	    self.ival = x
+	elif isinstance(x, long):
+	    self.ival = (int(x & 0xffffffff))
+	elif isinstance(x, str):
+	    self.ival = 0
+	    sh = 24
+	    bytes = x.split(".", 4)
+	    for b in bytes:
+		self.ival |= (int(b) << sh)
+		sh -= 8
+	else:
+	    self.ival = 0
+    
+    def __str__(self):
+	return "%d.%d.%d.%d" % ((self.ival & 0xff000000) >> 24,
+		(self.ival & 0x00ff0000) >> 16, 
+		(self.ival & 0x0000ff00) >> 8,
+		self.ival & 0x000000ff)
+
+    def __add__(self, i):
+	if isinstance(i, int): return ip_addr(self.ival + i)
+	elif isinstance(i, ip_addr): return ip_addr(self.ival + i.ival)
+	else: raise ValueError("IP addrs must operate on IP addrs or ints")
+
+    def __sub__(self, i):
+	if isinstance(i, int): return ip_addr(self.ival - i)
+	elif isinstance(i, ip_addr): return ip_addr(self.ival - i.ival)
+	else: raise ValueError("IP addrs must operate on IP addrs or ints")
+
+    def __and__(self, i):
+	if isinstance(i, int): return ip_addr(self.ival & i)
+	elif isinstance(i, ip_addr): return ip_addr(self.ival & i.ival)
+	else: raise ValueError("IP addrs must operate on IP addrs or ints")
+
+    def __or__(self, i):
+	if isinstance(i, int): return ip_addr(self.ival | i)
+	elif isinstance(i, ip_addr): return ip_addr(self.ival | i.ival)
+	else: raise ValueError("IP addrs must operate on IP addrs or ints")
+
+    def __xor__(self, i):
+	if isinstance(i, int): return ip_addr(self.ival ^ i)
+	elif isinstance(i, ip_addr): return ip_addr(self.ival ^ i.ival)
+	else: raise ValueError("IP addrs must operate on IP addrs or ints")
+
+    def __radd__(self, i):
+	return self.__add__(i)
+
+    def __rsub__(self, i):
+	return self.__sub__(i)
+
+    def __rand__(self, i):
+	return self.__and__(i)
+
+    def __ror__(self, i):
+	return self.__or__(i)
+
+    def __rxor__(self, i):
+	return self.__xor__(i)
+
+    def __int__(self):
+	return self.ival
+
+    def __oct__(self):
+	return oct(self.ival)
+
+    def __hex__(self):
+	return hex(self.ival)
Index: fedd/Makefile/federation/ip_allocator.py
===================================================================
--- fedd/Makefile/federation/ip_allocator.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/ip_allocator.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,87 @@
+#!/usrl/ocal/bin/python
+
+class ip_allocator:
+    def __init__(self, base=0, size=4294967296, min_alloc=256):
+
+        self.base = base
+        self.size = size
+        self.min_alloc = min_alloc
+
+        i = self.min_alloc
+        self.free = { }
+        self.allocated = { }
+        while i < self.size:
+            self.free[i] = set()
+            self.allocated[i] = set()
+            i *= 2
+        self.free[self.size] = set([self.base])
+        self.allocated[self.size] = set()
+
+    def split(self, base, size):
+        return ( base, base + size)
+
+    def make_chunk(self, size):
+        if not self.free[size * 2]:
+            if size < self.size:
+                if not self.make_chunk(size * 2):
+                    return False
+            else:
+                return False
+        base = min(self.free[size*2])
+        self.free[size*2].discard(base)
+
+        for b in self.split(base, size):
+            self.free[size].add(b)
+        return True
+
+    def allocate(self, size):
+        # Make sure size is a power of 2 at least as big as minalloc
+        ns = self.min_alloc
+        while ns < self.size:
+            if ns == size:
+                break
+            if ns > size:
+                size = ns
+                break
+            ns *= 2
+        else:
+            size = self.size
+
+        if self.free[size] or self.make_chunk(size):
+            rv = min(self.free[size])
+            self.free[size].discard(rv)
+            self.allocated[size].add(rv)
+            return (rv, size)
+        else:
+            return None
+
+    def deallocate(self, base, size):
+        if base not in self.allocated[size]:
+            raise KeyError("Not allocated")
+        if base % (size * 2) == 0:
+            if base + size in self.free[size]:
+                self.allocated[size].discard(base)
+                self.allocated[size * 2].add(base)
+                self.free[size].discard(base+size)
+                self.deallocate(base, size *2)
+            else:
+                self.allocated[size].discard(base)
+                self.free[size].add(base)
+        else:
+            if base - size in self.free[size]:
+                self.allocated[size].discard(base)
+                self.allocated[size * 2].add(base - size)
+                self.free[size].discard(base - size)
+                self.deallocate(base-size, size * 2)
+            else:
+                self.allocated[size].discard(base)
+                self.free[size].add(base)
+        return True
+
+    def __str__(self):
+        rv = ""
+        i=self.size
+        while i >= self.min_alloc:
+            rv += "%d: %s\n" % (i, self.free[i])
+            i /= 2
+        return rv
Index: fedd/Makefile/federation/list_log.py
===================================================================
--- fedd/Makefile/federation/list_log.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/list_log.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,28 @@
+#!/usr/local/bin/python
+
+from threading import Lock
+
+class list_log:
+    """
+    Provide an interface that lets logger.StreamHandler s write to a list
+    of strings.
+    """
+    def __init__(self, l=[]):
+	"""
+	Link to an existing list or just create a log
+	"""
+	self.ll = l
+	self.lock = Lock()
+    def write(self, str):
+	"""
+	Add the string to the log.  Lock for consistency.
+	"""
+	self.lock.acquire()
+	self.ll.append(str)
+	self.lock.release()
+
+    def flush(self):
+	"""
+	No-op that StreamHandlers expect
+	"""
+	pass
Index: fedd/Makefile/federation/local_emulab_segment.py
===================================================================
--- fedd/Makefile/federation/local_emulab_segment.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/local_emulab_segment.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,117 @@
+#!/usr/local/bin/python
+
+import sys, os
+import re
+
+import tempfile
+import subprocess
+import logging 
+import time
+import signal
+
+import util
+
+from local_segment import local_segment
+from federation.emulab_segment import emulab_segment
+
+
+class start_segment(local_segment, emulab_segment):
+    def __init__(self, log=None, keyfile=None, debug=False, boss=None,
+	    cert=None):
+	local_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+	emulab_segment.__init__(self, boss=boss, cert=cert)
+	self.node = { }
+
+    def set_up_experiment_filespace(self, pid, eid, tmpdir):
+	# Configuration directories on this machine
+	proj_dir = "/proj/%s/exp/%s/tmp" % (pid, eid)
+	softdir = "/proj/%s/software/%s" % (pid, eid)
+	# Softwrae staging directory software dir
+	lsoftdir = "%s/software" % tmpdir
+
+	# Set up the experiment's file space
+	if not self.cmd_with_timeout("/bin/rm -rf %s" % proj_dir):
+	    return False
+	# Clear and create the software and configuration directories
+	if not self.cmd_with_timeout("/bin/rm -rf %s/*" % softdir):
+	    return False
+	if not self.cmd_with_timeout('mkdir -p %s' % proj_dir):
+	    return False
+	if os.path.isdir(lsoftdir):
+	    if not self.cmd_with_timeout('mkdir -p %s' % softdir):
+		return False
+
+	try:
+	    for f in os.listdir(tmpdir):
+		if not os.path.isdir("%s/%s" % (tmpdir, f)):
+		    self.copy_file("%s/%s" % (tmpdir, f), 
+			    "%s/%s" % (proj_dir, f))
+	    if os.path.isdir(lsoftdir):
+		for f in os.listdir(lsoftdir):
+		    if not os.path.isdir("%s/%s" % (lsoftdir, f)):
+			self.copy_file("%s/%s" % (lsoftdir, f), 
+				"%s/%s" % (softdir, f))
+	except EnvironmentError, e:
+	    self.log.error("Error copying file: %s" %e)
+	    return False
+
+	return True
+
+    def __call__(self, parent, eid, pid, user, tclfile, tmpdir, timeout=0):
+	"""
+	Start a sub-experiment on a federant.
+
+	Get the current state, modify or create as appropriate, ship data
+	and configs and start the experiment.  There are small ordering
+	differences based on the initial state of the sub-experiment.
+	"""
+
+	state = self.get_state(pid, eid)
+
+	if state == 'none':
+	    if not self.make_null_experiment(pid, eid, tmpdir):
+		return False
+
+	if not self.set_up_experiment_filespace(pid, eid, tmpdir):
+	    return False
+
+	# Put the file into a string to pass to emulab.
+	try:
+	    tcl = "".join([ l for l in open(tclfile,"r")])
+	except EnvironmentError, e:
+	    self.log.error("Can't read %s: %s" % (tclfile, e))
+	    return False
+	
+	# Stage the new configuration (active experiments will stay swapped
+	# in now)
+	if not self.modify_exp(pid, eid, tcl):
+	    self.log.error("modify failed")
+	    return False
+	# Active experiments are still swapped, this swaps the others in.
+	if state != 'active':
+	    if not self.swap_exp(pid, eid, 'in'):
+		return False
+	# Everything has gone OK.
+	self.get_mapping(pid,eid)
+	return True
+
+class stop_segment(local_segment, emulab_segment):
+    def __init__(self, log=None, keyfile=None, debug=False, boss=None, 
+	    cert=None):
+	local_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+	emulab_segment.__init__(self, boss=boss, cert=cert)
+
+    def __call__(self, parent, user, pid, eid):
+	"""
+	Stop a sub experiment by calling swapexp on the federant
+	"""
+	self.log.info("[stop_segment]: Stopping %s" % eid)
+	rv = False
+	try:
+	    # Clean out tar files: we've gone over quota in the past
+	    self.cmd_with_timeout("rm -rf /proj/%s/software/%s" % (pid, eid))
+	    rv = self.swap_exp(pid, eid, 'out')
+	except self.cmd_timeout:
+	    rv = False
+	return rv
+
Index: fedd/Makefile/federation/local_segment.py
===================================================================
--- fedd/Makefile/federation/local_segment.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/local_segment.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,86 @@
+#!/usr/local/bin/python
+
+import sys, os
+import re
+
+import tempfile
+import subprocess
+import logging 
+import time
+import signal
+
+import util
+
+class local_segment:
+    """
+    Base class for starting a segments on a local testbed accessed directly
+    through commands rather than ssh connections.  Code repository for
+    executing commands with timeouts and other utility functions.
+    """
+    class cmd_timeout(RuntimeError): pass
+
+    def __init__(self, log=None, keyfile=None, debug=False):
+	"""
+	Parameters are a logger to use for updates, an unused key file, and a
+	flag indicating that commands should not actually be carried out.
+	"""
+	self.log = log or logging.getLogger(\
+		'fedd.access.proxy_emulab_segment')
+	self.certfile = keyfile
+	self.debug = debug
+	self.cmd_timeout = local_segment.cmd_timeout
+
+    def copy_file(self, src, dest, size=1024):
+        """
+        Exceedingly simple file copy.
+        """
+	if not self.debug:
+	    util.copy_file(src, dest, size)
+	else:
+	    self.log.debug("Copy %s to %s" % (src, dest))
+
+    def cmd_with_timeout(self, cmd, wname=None, timeout=None):
+	"""
+	Run a command.  If debug is set, the action
+	is only logged.  Commands are run without stdin, to avoid stray
+	SIGTTINs. If timeout is given and the command runs longer, a
+	cmd_timeout exception is thrown.
+	"""
+
+	try:
+	    dnull = open("/dev/null", "w")
+	except EnvironmentError:
+	    self.log.debug("[cmd_with_timeout]: failed to open /dev/null " + \
+		    "for redirect")
+	    dnull = Null
+
+	self.log.debug("[cmd_with_timeout]: %s" % cmd)
+	if not self.debug:
+	    if dnull:
+		sub = subprocess.Popen(cmd, shell=True, stdout=dnull,
+			stderr=dnull, close_fds=True)
+	    else:
+		sub = subprocess.Popen(cmd, shell=True, close_fds=True)
+	    if timeout:
+		i = 0
+		rv = sub.poll()
+		while i < timeout:
+		    if rv is not None: break
+		    else:
+			time.sleep(1)
+			rv = sub.poll()
+			i += 1
+		else:
+		    self.log.debug("Process exceeded runtime: %s" % cmd)
+		    os.kill(sub.pid, signal.SIGKILL)
+		    raise self.cmd_timeout();
+		return rv == 0
+	    else:
+		return sub.wait() == 0
+	else:
+	    if timeout == 0:
+		self.log.debug("debug timeout raised on %s " % cmd)
+		raise self.cmd_timeout()
+	    else:
+		return True
+
Index: fedd/Makefile/federation/ns2topdl.py
===================================================================
--- fedd/Makefile/federation/ns2topdl.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/ns2topdl.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,144 @@
+#!/usr/local/bin/python
+
+import os,sys
+import subprocess
+import tempfile
+import logging
+import topdl
+
+from util import *
+from fedid import fedid
+from remote_service import xmlrpc_handler, soap_handler
+from service_error import *
+from authorizer import authorizer
+
+
+class nullHandler(logging.Handler):
+    def emit(self, record): pass
+
+fl = logging.getLogger("fedd.splitter")
+fl.addHandler(nullHandler())
+
+class ns2topdl_local:
+    def __init__(self, config=None, auth=None):
+	"""
+	Intialize the various attributes, most from the config object
+	"""
+	self.debug = config.getboolean("ns2topdl", "debug")
+	self.muxmax = config.getint("ns2topdl", "muxmax", 3)
+	self.tclsh = config.get("ns2topdl", "tclsh", 
+		"/usr/local/bin/otclsh")
+	self.tcl_splitter = config.get("ns2topdl", "tcl_splitter",
+		"/usr/testbed/lib/ns2ir/parse.tcl")
+	access_db = config.get("ns2topdl", "accessdb", None)
+	allow_any = config.getboolean("ns2topdl", "allow_any", False)
+
+	self.log = logging.getLogger("fedd.ns2topdl")
+	set_log_level(config, "ns2topdl", self.log)
+	self.trace_file = sys.stderr
+
+	set_log_level(config, "ns2topdl", self.log)
+
+	if auth:
+	    self.auth= auth
+	else:
+	    self.auth = authorizer()
+	    self.log.warning("[ns2topdl] No authorizer passed in, " +\
+		    "using local one")
+
+	if access_db and allow_any: 
+	    raise service_error(service_error.internal, 
+		    "Cannot specify both an access database and allow_any " +\
+			    "for ns2topdl")
+	
+	if access_db:
+	    try:
+		read_simple_accessdb(access_db, self.auth, 'ns2topdl')
+	    except EnvironmentError, e:
+		raise service_error(service_error.internal,
+			"Error reading accessDB %s: %s" % (access_db, e))
+	    except ValueError:
+		raise service_error(service_error.internal, "%s" % e)
+	elif allow_any:
+	    auth.set_global_attribute("ns2topdl")
+
+
+	# Dispatch tables
+	self.soap_services = {\
+		'Ns2Topdl': soap_handler("Ns2Topdl", self.run_ns2topdl),
+	}
+
+	self.xmlrpc_services = {\
+		'Ns2Topdl': xmlrpc_handler('Ns2Topdl', self.run_ns2topdl),
+	}
+
+    def run_ns2topdl(self, req, fid):
+	"""
+	The external interface to tcl to topdl translation.
+
+	Creates a working directory, translates the incoming description using
+	the tcl script.
+	"""
+
+	if not self.auth.check_attribute(fid, 'ns2topdl'):
+	    raise service_error(service_error.access, "Access Denied")
+
+	try:
+	    tmpdir = tempfile.mkdtemp(prefix="ns2topdl-")
+	except EnvironmentError:
+	    raise service_error(service_error.internal, "Cannot create tmp dir")
+
+	tclfile = tmpdir + "/experiment.tcl"
+	pid = "dummy"
+	gid = "dummy"
+	eid = "dummy"
+	master = "dummy"
+
+	req = req.get('Ns2TopdlRequestBody', None)
+	if not req:
+	    raise service_error(service_error.req,
+		    "Bad request format (no Ns2TopdlRequestBody)")
+	# The tcl parser needs to read a file so put the content into that file
+	descr=req.get('description', None)
+	if descr:
+	    file_content=descr.get('ns2description', None)
+	    if file_content:
+		try:
+		    f = open(tclfile, 'w')
+		    f.write(file_content)
+		    f.close()
+		except EnvironmentError:
+		    raise service_error(service_error.internal,
+			    "Cannot write temp experiment description")
+	    else:
+		raise service_error(service_error.req, 
+			"Only ns2descriptions supported")
+	else:
+	    raise service_error(service_error.req, "No description")
+
+	
+	tclcmd = [self.tclsh, self.tcl_splitter, '-t', '-x', 
+	    str(self.muxmax), '-m', master]
+	tclcmd.extend([pid, gid, eid, tclfile])
+	self.log.debug("Calling splitter %s" % " ".join(tclcmd))
+	tclparser = subprocess.Popen(tclcmd, stdout=subprocess.PIPE)
+
+	top = topdl.topology_from_xml(file=tclparser.stdout, top="experiment")
+
+	# Walk up tmpdir, deleting as we go
+	for path, dirs, files in os.walk(tmpdir, topdown=False):
+	    for f in files:
+		os.remove(os.path.join(path, f))
+	    for d in dirs:
+		os.rmdir(os.path.join(path, d))
+	os.rmdir(tmpdir)
+
+	resp = { 
+		'experimentdescription':  { 
+		    'topdldescription': top.to_dict(),
+		    }
+		}
+
+	return resp
+
+
Index: fedd/Makefile/federation/parse_detail.py
===================================================================
--- fedd/Makefile/federation/parse_detail.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/parse_detail.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,39 @@
+#!/usr/local/bin/python
+
+from fedd_services import *
+
+from ZSI.TC import QName, String, URI, AnyElement, UNBOUNDED, Any
+from ZSI.wstools.Namespaces import SOAP
+from ZSI.fault import FaultType
+
+
+# This is a bit of minimal trickery to integrate detail parsing of SOAP faults.
+# ZSI expects detail fields to parse to a class with an "any" attribute.  This
+# little class has such an attribute pointed at itself, so when
+# ZSI.fault.FaultFromMessage extracts that field, it places this object into
+# the new Fault object.  One can also get around this using an extra XML field
+# in the message, parsing the wrapper to the any attribute and then actial body
+# to the class in the any attribute, but this does not perturb the WSDL.
+class detail_faultType:
+    def __init__(self):
+	self.FeddFaultBody = None
+	self.any = self
+    def get_element_FeddFaultBody(self):
+	return self.FeddFaultBody
+
+# Override ZSI.fault.FaultType.typecode - give a parser that speaks our
+# language.
+FaultType.typecode = \
+    Struct(FaultType,
+    [QName(pname='faultcode'),
+    String(pname='faultstring'),
+    URI(pname=(SOAP.ENV,'faultactor'), minOccurs=0),
+    Struct(pname='detail', aname='detail', 
+	ofwhat=[ns0.faultType_Def((ns0.faultType_Def.schema, "FeddFaultBody"))],
+	pyclass=detail_faultType, minOccurs=0),
+    ],  
+    pname=(SOAP.ENV,'Fault'),
+    inline=True,
+    hasextras=0,
+)
+
Index: fedd/Makefile/federation/protogeni_access.py
===================================================================
--- fedd/Makefile/federation/protogeni_access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/protogeni_access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,1478 @@
+#!/usr/local/bin/python
+
+import os,sys
+import stat # for chmod constants
+import re
+import time
+import string
+import copy
+import pickle
+import logging
+import subprocess
+import random
+import traceback
+import xml.parsers.expat
+
+from threading import Thread, Timer, Lock
+
+from util import *
+from access_project import access_project
+from fedid import fedid, generate_fedid
+from authorizer import authorizer
+from service_error import service_error
+from remote_service import xmlrpc_handler, soap_handler, service_caller
+
+import httplib
+import tempfile
+from urlparse import urlparse
+
+from access import access_base
+from protogeni_proxy import protogeni_proxy
+from geniapi_proxy import geniapi_proxy
+
+import topdl
+import list_log
+
+
+# Make log messages disappear if noone configures a fedd logger
+class nullHandler(logging.Handler):
+    def emit(self, record): pass
+
+fl = logging.getLogger("fedd.access")
+fl.addHandler(nullHandler())
+
+class access(access_base):
+    """
+    The implementation of access control based on mapping users to projects.
+
+    Users can be mapped to existing projects or have projects created
+    dynamically.  This implements both direct requests and proxies.
+    """
+
+    def __init__(self, config=None, auth=None):
+        """
+        Initializer.  Pulls parameters out of the ConfigParser's access section.
+        """
+
+	access_base.__init__(self, config, auth)
+
+        self.domain = config.get("access", "domain")
+	self.userconfdir = config.get("access","userconfdir")
+	self.userconfcmd = config.get("access","userconfcmd")
+	self.userconfurl = config.get("access","userconfurl")
+	self.federation_software = config.get("access", "federation_software")
+	self.portal_software = config.get("access", "portal_software")
+	self.ssh_port = config.get("access","ssh_port") or "22"
+	self.sshd = config.get("access","sshd")
+	self.sshd_config = config.get("access", "sshd_config")
+	self.access_type = config.get("access", "type")
+	self.staging_dir = config.get("access", "staging_dir") or "/tmp"
+	self.staging_host = config.get("access", "staging_host") \
+		or "ops.emulab.net"
+	self.local_seer_software = config.get("access", "local_seer_software")
+	self.local_seer_image = config.get("access", "local_seer_image")
+	self.local_seer_start = config.get("access", "local_seer_start")
+   
+	self.dragon_endpoint = config.get("access", "dragon")
+	self.dragon_vlans = config.get("access", "dragon_vlans")
+	self.deter_internal = config.get("access", "deter_internal")
+
+	self.tunnel_config = config.getboolean("access", "tunnel_config")
+	self.portal_command = config.get("access", "portal_command")
+	self.portal_image = config.get("access", "portal_image")
+	self.portal_type = config.get("access", "portal_type") or "pc"
+	self.portal_startcommand = config.get("access", "portal_startcommand")
+	self.node_startcommand = config.get("access", "node_startcommand")
+
+	self.federation_software = self.software_list(self.federation_software)
+	self.portal_software = self.software_list(self.portal_software)
+	self.local_seer_software = self.software_list(self.local_seer_software)
+
+	self.renewal_interval = config.get("access", "renewal") or (3 * 60 )
+	self.renewal_interval = int(self.renewal_interval) * 60
+
+	self.ch_url = config.get("access", "ch_url")
+	self.sa_url = config.get("access", "sa_url")
+	self.cm_url = config.get("access", "cm_url")
+
+        self.restricted = [ ]
+
+	# read_state in the base_class
+	self.state_lock.acquire()
+	for a  in ('allocation', 'projects', 'keys', 'types'):
+	    if a not in self.state:
+		self.state[a] = { }
+	self.allocation = self.state['allocation']
+	self.projects = self.state['projects']
+	self.keys = self.state['keys']
+	self.types = self.state['types']
+	# Add the ownership attributes to the authorizer.  Note that the
+	# indices of the allocation dict are strings, but the attributes are
+	# fedids, so there is a conversion.
+	for k in self.state.get('allocation', {}).keys():
+	    for o in self.state['allocation'][k].get('owners', []):
+		self.auth.set_attribute(o, fedid(hexstr=k))
+	    self.auth.set_attribute(fedid(hexstr=k),fedid(hexstr=k))
+
+	self.state_lock.release()
+
+
+        self.log = logging.getLogger("fedd.access")
+        set_log_level(config, "access", self.log)
+
+	self.access = { }
+        if config.has_option("access", "accessdb"):
+            self.read_access(config.get("access", "accessdb"), 
+		    access_obj=self.make_access_info)
+
+	self.lookup_access = self.lookup_access_base
+
+	api = config.get("access", "api") or "protogeni"
+	if api == "protogeni":
+	    self.api_proxy = protogeni_proxy
+	elif api == "geniapi":
+	    self.api_proxy = geniapi_proxy
+	else:
+	    self.log.debug("Unknown interface, using protogeni")
+	    self.api_proxy = protogeni_proxy
+
+	self.call_SetValue = service_caller('SetValue')
+	self.call_GetValue = service_caller('GetValue')
+	self.exports = {
+		'local_seer_control': self.export_local_seer,
+		'seer_master': self.export_seer_master,
+		'hide_hosts': self.export_hide_hosts,
+		}
+
+	if not self.local_seer_image or not self.local_seer_software or \
+		not self.local_seer_start:
+	    if 'local_seer_control' in self.exports:
+		del self.exports['local_seer_control']
+
+	if not self.local_seer_image or not self.local_seer_software or \
+		not self.seer_master_start:
+	    if 'seer_master' in self.exports:
+		del self.exports['seer_master']
+
+	self.RenewSlices()
+
+        self.soap_services = {\
+            'RequestAccess': soap_handler("RequestAccess", self.RequestAccess),
+            'ReleaseAccess': soap_handler("ReleaseAccess", self.ReleaseAccess),
+            'StartSegment': soap_handler("StartSegment", self.StartSegment),
+            'TerminateSegment': soap_handler("TerminateSegment", 
+		self.TerminateSegment),
+            }
+        self.xmlrpc_services =  {\
+            'RequestAccess': xmlrpc_handler('RequestAccess',
+                self.RequestAccess),
+            'ReleaseAccess': xmlrpc_handler('ReleaseAccess',
+                self.ReleaseAccess),
+            'StartSegment': xmlrpc_handler("StartSegment", self.StartSegment),
+            'TerminateSegment': xmlrpc_handler('TerminateSegment',
+                self.TerminateSegment),
+            }
+
+    @staticmethod
+    def make_access_info(s):
+	"""
+	Split a string of the form (id, id, id, id) ito its constituent tuples
+	and return them as a tuple.  Use to import access info from the
+	access_db.
+	"""
+
+	ss = s.strip()
+	if ss.startswith('(') and ss.endswith(')'):
+	    l = [ s.strip() for s  in ss[1:-1].split(",")]
+	    if len(l) == 4:
+		return tuple(l)
+	    else:
+		raise self.parse_error(
+			"Exactly 4 elements in access info required")
+	else:
+	    raise self.parse_error("Expecting parenthezied values")
+
+
+    def get_handler(self, path, fid):
+	self.log.info("Get handler %s %s" % (path, fid))
+	if self.auth.check_attribute(fid, path) and self.userconfdir:
+	    return ("%s/%s" % (self.userconfdir, path), "application/binary")
+	else:
+	    return (None, None)
+
+    def build_access_response(self, alloc_id, services):
+        """
+        Create the SOAP response.
+
+        Build the dictionary description of the response and use
+        fedd_utils.pack_soap to create the soap message.  ap is the allocate
+        project message returned from a remote project allocation (even if that
+        allocation was done locally).
+        """
+        # Because alloc_id is already a fedd_services_types.IDType_Holder,
+        # there's no need to repack it
+        msg = { 
+                'allocID': alloc_id,
+		'fedAttr': [
+		    { 'attribute': 'domain', 'value': self.domain } , 
+		]
+            }
+	if self.dragon_endpoint:
+	    msg['fedAttr'].append({'attribute': 'dragon',
+		'value': self.dragon_endpoint})
+	if self.deter_internal:
+	    msg['fedAttr'].append({'attribute': 'deter_internal',
+		'value': self.deter_internal})
+	#XXX: ??
+	if self.dragon_vlans:
+	    msg['fedAttr'].append({'attribute': 'vlans',
+		'value': self.dragon_vlans})
+
+	if services:
+	    msg['service'] = services
+        return msg
+
+    def RequestAccess(self, req, fid):
+        """
+        Handle the access request.
+        """
+
+        # The dance to get into the request body
+        if req.has_key('RequestAccessRequestBody'):
+            req = req['RequestAccessRequestBody']
+        else:
+            raise service_error(service_error.req, "No request!?")
+
+        if req.has_key('destinationTestbed'):
+            dt = unpack_id(req['destinationTestbed'])
+
+	# Request for this fedd
+	found, match = self.lookup_access(req, fid)
+	services, svc_state = self.export_services(req.get('service',[]),
+		None, None)
+	# keep track of what's been added
+	allocID, alloc_cert = generate_fedid(subj="alloc", log=self.log)
+	aid = unicode(allocID)
+
+	self.state_lock.acquire()
+	self.allocation[aid] = { }
+	# The protoGENI certificate
+	self.allocation[aid]['credentials'] = found
+	# The list of owner FIDs
+	self.allocation[aid]['owners'] = [ fid ]
+	self.write_state()
+	self.state_lock.release()
+	self.auth.set_attribute(fid, allocID)
+	self.auth.set_attribute(allocID, allocID)
+
+	try:
+	    f = open("%s/%s.pem" % (self.certdir, aid), "w")
+	    print >>f, alloc_cert
+	    f.close()
+	except EnvironmentError, e:
+	    raise service_error(service_error.internal, 
+		    "Can't open %s/%s : %s" % (self.certdir, aid, e))
+	return self.build_access_response({ 'fedid': allocID }, None)
+
+
+    def ReleaseAccess(self, req, fid):
+        # The dance to get into the request body
+        if req.has_key('ReleaseAccessRequestBody'):
+            req = req['ReleaseAccessRequestBody']
+        else:
+            raise service_error(service_error.req, "No request!?")
+
+	# Local request
+	try:
+	    if req['allocID'].has_key('localname'):
+		auth_attr = aid = req['allocID']['localname']
+	    elif req['allocID'].has_key('fedid'):
+		aid = unicode(req['allocID']['fedid'])
+		auth_attr = req['allocID']['fedid']
+	    else:
+		raise service_error(service_error.req,
+			"Only localnames and fedids are understood")
+	except KeyError:
+	    raise service_error(service_error.req, "Badly formed request")
+
+	self.log.debug("[access] deallocation requested for %s", aid)
+	if not self.auth.check_attribute(fid, auth_attr):
+	    self.log.debug("[access] deallocation denied for %s", aid)
+	    raise service_error(service_error.access, "Access Denied")
+
+	self.state_lock.acquire()
+	if self.allocation.has_key(aid):
+	    self.log.debug("Found allocation for %s" %aid)
+	    del self.allocation[aid]
+	    self.write_state()
+	    self.state_lock.release()
+	    # And remove the access cert
+	    cf = "%s/%s.pem" % (self.certdir, aid)
+	    self.log.debug("Removing %s" % cf)
+	    os.remove(cf)
+	    return { 'allocID': req['allocID'] } 
+	else:
+	    self.state_lock.release()
+	    raise service_error(service_error.req, "No such allocation")
+
+    def manifest_to_dict(self, manifest, ignore_debug=False):
+	"""
+	Turn the manifest into a dict were each virtual nodename (i.e. the
+	topdl name) has an entry with the allocated machine in hostname and the
+	interfaces in 'interfaces'.  I love having XML parser code lying
+	around.
+	"""
+	if self.create_debug and not ignore_debug: 
+	    self.log.debug("Returning null manifest dict")
+	    return { }
+
+	# The class allows us to keep a little state - the dict under
+	# consteruction and the current entry in that dict for the interface
+	# element code.
+	class manifest_parser:
+	    def __init__(self):
+		self.d = { }
+		self.current_key=None
+
+	    # If the element is a node, create a dict entry for it.  If it's an
+	    # interface inside a node, add an entry in the interfaces list with
+	    # the virtual name and component id.
+	    def start_element(self, name, attrs):
+		if name == 'node':
+		    self.current_key = attrs.get('virtual_id',"")
+		    if self.current_key:
+			self.d[self.current_key] = {
+				'hostname': attrs.get('hostname', None),
+				'interfaces': { }
+				}
+		elif name == 'interface' and self.current_key:
+		    self.d[self.current_key]['interfaces']\
+			    [attrs.get('virtual_id','')] = \
+			    attrs.get('component_id', None)
+	    #  When a node is finished, clear current_key
+	    def end_element(self, name):
+		if name == 'node': self.current_key = None
+
+	node = { }
+
+	mp = manifest_parser()
+	p = xml.parsers.expat.ParserCreate()
+	# These are bound to the class we just created
+	p.StartElementHandler = mp.start_element
+	p.EndElementHandler = mp.end_element
+
+	p.Parse(manifest)
+	# Make the node dict that the callers expect
+	for k in mp.d:
+	    node[k] = mp.d.get('hostname', '')
+	return mp.d
+
+    def fake_manifest(self, topo):
+	"""
+	Fake the output of manifest_to_dict with a bunch of generic node an
+	interface names, for debugging.
+	"""
+	node = { }
+	for i, e in enumerate([ e for e in topo.elements \
+		if isinstance(e, topdl.Computer)]):
+	    node[e.name] = { 
+		    'hostname': "node%03d" % i,
+		    'interfaces': { }
+		    }
+	    for j, inf in enumerate(e.interface):
+		node[e.name]['interfaces'][inf.name] = 'eth%d' % j
+
+	return node
+
+
+    def generate_portal_configs(self, topo, pubkey_base, 
+	    secretkey_base, tmpdir, leid, connInfo, services, nodes):
+
+	def conninfo_to_dict(key, info):
+	    """
+	    Make a cpoy of the connection information about key, and flatten it
+	    into a single dict by parsing out any feddAttrs.
+	    """
+
+	    rv = None
+	    for i in info:
+		if key == i.get('portal', "") or \
+			key in [e.get('element', "") \
+			for e in i.get('member', [])]:
+		    rv = i.copy()
+		    break
+
+	    else:
+		return rv
+
+	    if 'fedAttr' in rv:
+		for a in rv['fedAttr']:
+		    attr = a.get('attribute', "")
+		    val = a.get('value', "")
+		    if attr and attr not in rv:
+			rv[attr] = val
+		del rv['fedAttr']
+	    return rv
+
+	# XXX: un hardcode this
+	def client_null(f, s):
+	    print >>f, "Service: %s" % s['name']
+	
+	def client_seer_master(f, s):
+	    print >>f, 'PortalAlias: seer-master'
+
+	def client_smb(f, s):
+	    print >>f, "Service: %s" % s['name']
+	    smbshare = None
+	    smbuser = None
+	    smbproj = None
+	    for a in s.get('fedAttr', []):
+		if a.get('attribute', '') == 'SMBSHARE':
+		    smbshare = a.get('value', None)
+		elif a.get('attribute', '') == 'SMBUSER':
+		    smbuser = a.get('value', None)
+		elif a.get('attribute', '') == 'SMBPROJ':
+		    smbproj = a.get('value', None)
+
+	    if all((smbshare, smbuser, smbproj)):
+		print >>f, "SMBshare: %s" % smbshare
+		print >>f, "ProjectUser: %s" % smbuser
+		print >>f, "ProjectName: %s" % smbproj
+
+	def client_hide_hosts(f, s):
+	    for a in s.get('fedAttr', [ ]):
+		if a.get('attribute', "") == 'hosts':
+		    print >>f, 'Hide: %s' % a.get('value', "")
+
+	client_service_out = {
+		'SMB': client_smb,
+		'tmcd': client_null,
+		'seer': client_null,
+		'userconfig': client_null,
+		'project_export': client_null,
+		'seer_master': client_seer_master,
+		'hide_hosts': client_hide_hosts,
+	    }
+
+	def client_seer_master_export(f, s):
+	    print >>f, "AddedNode: seer-master"
+
+	def client_seer_local_export(f, s):
+	    print >>f, "AddedNode: control"
+
+	client_export_service_out = {
+		'seer_master': client_seer_master_export,
+		'local_seer_control': client_seer_local_export,
+	    }
+
+	def server_port(f, s):
+	    p = urlparse(s.get('server', 'http://localhost'))
+	    print >>f, 'port: remote:%s:%s:%s' % (p.port, p.hostname, p.port) 
+
+	def server_null(f,s): pass
+
+	def server_seer(f, s):
+	    print >>f, 'seer: true'
+
+	server_service_out = {
+		'SMB': server_port,
+		'tmcd': server_port,
+		'userconfig': server_null,
+		'project_export': server_null,
+		'seer': server_seer,
+		'seer_master': server_port,
+		'hide_hosts': server_null,
+	    }
+	# XXX: end un hardcode this
+
+
+	seer_out = False
+	client_out = False
+	for e in [ e for e in topo.elements \
+		if isinstance(e, topdl.Computer) and e.get_attribute('portal')]:
+	    myname = e.name
+	    type = e.get_attribute('portal_type')
+
+	    info = conninfo_to_dict(myname, connInfo)
+
+	    if not info:
+		raise service_error(service_error.req,
+			"No connectivity info for %s" % myname)
+
+	    # Translate to physical name (ProtoGENI doesn't have DNS)
+	    physname = nodes.get(myname, { }).get('hostname', None)
+	    peer = info.get('peer', "")
+	    ldomain = self.domain
+	    ssh_port = info.get('ssh_port', 22)
+
+	    # Collect this for the client.conf file
+	    if 'masterexperiment' in info:
+		mproj, meid = info['masterexperiment'].split("/", 1)
+
+	    active = info.get('active', 'False')
+
+	    if type in ('control', 'both'):
+		testbed = e.get_attribute('testbed')
+		control_gw = myname
+
+	    cfn = "%s/%s.gw.conf" % (tmpdir, myname.lower())
+	    tunnelconfig = self.tunnel_config
+	    try:
+		f = open(cfn, "w")
+		if active == 'True':
+		    print >>f, "active: True"
+		    print >>f, "ssh_port: %s" % ssh_port
+		    if type in ('control', 'both'):
+			for s in [s for s in services \
+				if s.get('name', "") in self.imports]:
+			    server_service_out[s['name']](f, s)
+
+		if tunnelconfig:
+		    print >>f, "tunnelip: %s" % tunnelconfig
+		print >>f, "peer: %s" % peer.lower()
+		print >>f, "ssh_pubkey: /usr/local/federation/etc/%s" % \
+			pubkey_base
+		print >>f, "ssh_privkey: /usr/local/federation/etc/%s" % \
+			secretkey_base
+		f.close()
+	    except EnvironmentError, e:
+		raise service_error(service_error.internal,
+			"Can't write protal config %s: %s" % (cfn, e))
+
+	# Done with portals, write the client config file.
+	try:
+	    f = open("%s/client.conf" % tmpdir, "w")
+	    if control_gw:
+		print >>f, "ControlGateway: %s" % physname.lower()
+	    for s in services:
+		if s.get('name',"") in self.imports and \
+			s.get('visibility','') == 'import':
+		    client_service_out[s['name']](f, s)
+		if s.get('name', '') in self.exports and \
+			s.get('visibility', '') == 'export' and \
+			s['name'] in client_export_service_out:
+		    client_export_service_out[s['name']](f, s)
+	    # Seer uses this.
+	    if mproj and meid:
+		print >>f, "ExperimentID: %s/%s" % (mproj, meid)
+	    f.close()
+	except EnvironmentError, e:
+	    raise service_error(service_error.internal,
+		    "Cannot write client.conf: %s" %s)
+
+
+
+    def export_store_info(self, cf, nodes, ssh_port, connInfo):
+	"""
+	For the export requests in the connection info, install the peer names
+	at the experiment controller via SetValue calls.
+	"""
+
+	for c in connInfo:
+	    for p in [ p for p in c.get('parameter', []) \
+		    if p.get('type', '') == 'output']:
+
+		if p.get('name', '') == 'peer':
+		    k = p.get('key', None)
+		    surl = p.get('store', None)
+		    if surl and k and k.index('/') != -1:
+			if self.create_debug:
+			    req = { 'name': k, 'value': 'debug' }
+			    self.call_SetValue(surl, req, cf)
+			else:
+			    n = nodes.get(k[k.index('/')+1:], { })
+			    value = n.get('hostname', None)
+			    if value:
+				req = { 'name': k, 'value': value }
+				self.call_SetValue(surl, req, cf)
+			    else:
+				self.log.error("No hostname for %s" % \
+					k[k.index('/'):])
+		    else:
+			self.log.error("Bad export request: %s" % p)
+		elif p.get('name', '') == 'ssh_port':
+		    k = p.get('key', None)
+		    surl = p.get('store', None)
+		    if surl and k:
+			req = { 'name': k, 'value': ssh_port }
+			self.call_SetValue(surl, req, cf)
+		    else:
+			self.log.error("Bad export request: %s" % p)
+		else:
+
+		    self.log.error("Unknown export parameter: %s" % \
+			    p.get('name'))
+		    continue
+
+    def write_node_config_script(self, elem, node, user, pubkey,
+	    secretkey, stagingdir, tmpdir):
+	"""
+	Write out the configuration script that is to run on the node
+	represented by elem in the topology.  This is called
+	once per node to configure.
+	"""
+	# These little functions/functors just make things more readable.  Each
+	# one encapsulates a small task of copying software files or installing
+	# them.
+	class stage_file_type:
+	    """
+	    Write code copying file sfrom the staging host to the host on which
+	    this will run.
+	    """
+	    def __init__(self, user, host, stagingdir):
+		self.user = user
+		self.host = host
+		self.stagingdir = stagingdir
+		self.scp = "/usr/bin/scp -i .ssh/id_rsa -o " + \
+			"'ForwardX11 no' -o 'StrictHostKeyChecking no' "
+
+	    def __call__(self, script, file, dest="."):
+		# If the file is a full pathname, do not use stagingdir
+		if file.find('/') == -1:
+		    file = "%s/%s" % (self.stagingdir, file)
+		print >>script, "%s %s@%s:%s %s" % \
+			(self.scp, self.user, self.host, file, dest)
+
+	def install_tar(script, loc, base):
+	    """
+	    Print code to script to install a tarfile in loc.
+	    """
+	    tar = "/bin/tar"
+	    mkdir="/bin/mkdir"
+
+	    print >>script, "%s -p %s" % (mkdir, loc)
+	    print >>script, "%s -C %s -xzf %s" % (tar, loc, base)
+
+	def install_rpm(script, base):
+	    """
+	    Print code to script to install an rpm
+	    """
+	    rpm = "/bin/rpm"
+	    print >>script, "%s --install %s" % (rpm, base)
+
+	ifconfig = "/sbin/ifconfig"
+	stage_file = stage_file_type(user, self.staging_host, stagingdir)
+	pname = node.get('hostname', None)
+	fed_dir = "/usr/local/federation"
+	fed_etc_dir = "%s/etc" % fed_dir
+	fed_bin_dir = "%s/bin" % fed_dir
+	fed_lib_dir = "%s/lib" % fed_dir
+
+	if pname:
+	    sfile = "%s/%s.startup" % (tmpdir, pname)
+	    script = open(sfile, "w")
+	    # Reset the interfaces to the ones in the topo file
+	    for i in [ i for i in elem.interface \
+		    if not i.get_attribute('portal')]:
+		pinf = node['interfaces'].get(i.name, None)
+		addr = i.get_attribute('ip4_address') 
+		netmask = i.get_attribute('ip4_netmask') or '255.255.255.0'
+		if pinf and addr:
+		    print >>script, \
+			    "%s %s %s netmask %s"  % \
+			    (ifconfig, pinf, addr, netmask)
+		else:
+		    self.log.error("Missing interface or address for %s" \
+			    % i.name)
+		
+	    for l, f in self.federation_software:
+		base = os.path.basename(f)
+		stage_file(script, base)
+		if l: install_tar(script, l, base)
+		else: install_rpm(script, base)
+
+	    for s in elem.software:
+		s_base = s.location.rpartition('/')[2]
+		stage_file(script, s_base)
+		if s.install: install_tar(script, s.install, s_base)
+		else: install_rpm(script, s_base)
+
+	    for f in ('hosts', pubkey, secretkey, 'client.conf', 
+		    'userconf'):
+		stage_file(script, f, fed_etc_dir)
+	    if self.sshd:
+		stage_file(script, self.sshd, fed_bin_dir)
+	    if self.sshd_config:
+		stage_file(script, self.sshd_config, fed_etc_dir)
+
+	    # Look in tmpdir to get the names.  They've all been copied
+	    # into the (remote) staging dir
+	    if os.access("%s/%s.gw.conf" % (tmpdir, elem.name), os.R_OK):
+		stage_file(script, "%s.gw.conf" % elem.name, fed_etc_dir)
+
+	    # Done with staging, remove the identity used to stage
+	    print >>script, "#/bin/rm .ssh/id_rsa"
+
+	    # Start commands
+	    if elem.get_attribute('portal') and self.portal_startcommand:
+		# Install portal software
+		for l, f in self.portal_software:
+		    base = os.path.basename(f)
+		    stage_file(script, base)
+		    if l: install_tar(script, l, base)
+		    else: install_rpm(script, base)
+
+		# Portals never have a user-specified start command
+		print >>script, self.portal_startcommand
+	    elif self.node_startcommand:
+		# XXX: debug
+		print >>script, "sudo perl -I%s %s/import_key.pl /users/%s/.ssh/authorized_keys /root/.ssh/authorized_keys" % (fed_lib_dir, fed_bin_dir, user)
+		# XXX: debug
+		if elem.get_attribute('startup'):
+		    print >>script, "%s \\$USER '%s'" % \
+			    (self.node_startcommand,
+				    elem.get_attribute('startup'))
+		else:
+		    print >>script, self.node_startcommand
+	    script.close()
+	    return sfile, pname
+	else:
+	    return None, None
+
+
+    def configure_nodes(self, segment_commands, topo, nodes, user, 
+	    pubkey, secretkey, stagingdir, tmpdir):
+	"""
+	For each node in the topology, generate a script file that copies
+	software onto it and installs it in the proper places and then runs the
+	startup command (including the federation commands.
+	"""
+
+
+
+	for e in [ e for e in topo.elements if isinstance(e, topdl.Computer)]:
+	    vname = e.name
+	    sfile, pname = self.write_node_config_script(e,
+		    nodes.get(vname, { }),
+		    user, pubkey, secretkey, stagingdir, tmpdir)
+	    if sfile:
+		if not segment_commands.scp_file(sfile, user, pname):
+		    self.log.error("Could not copy script to %s" % pname)
+	    else:
+		self.log.error("Unmapped node: %s" % vname)
+
+    def start_node(self, user, host, node, segment_commands):
+	"""
+	Copy an identity to a node for the configuration script to be able to
+	import data and then run the startup script remotely.
+	"""
+	# Place an identity on the node so that the copying can succeed
+	segment_commands.scp_file( segment_commands.ssh_privkey_file,
+		user, node, ".ssh/id_rsa")
+	segment_commands.ssh_cmd(user, node, 
+		"sudo /bin/sh ./%s.startup &" % node)
+
+    def start_nodes(self, user, host, nodes, segment_commands):
+	"""
+	Start a thread to initialize each node and wait for them to complete.
+	Each thread runs start_node.
+	"""
+	threads = [ ]
+	for n in nodes:
+	    t = Thread(target=self.start_node, args=(user, host, n, 
+		segment_commands))
+	    t.start()
+	    threads.append(t)
+
+	done = [not t.isAlive() for t in threads]
+	while not all(done):
+	    self.log.info("Waiting for threads %s" % done)
+	    time.sleep(10)
+	    done = [not t.isAlive() for t in threads]
+
+    def set_up_staging_filespace(self, segment_commands, user, host,
+	    stagingdir):
+	"""
+	Set up teh staging area on the staging machine.  To reduce the number
+	of ssh commands, we compose a script and execute it remotely.
+	"""
+
+	self.log.info("[start_segment]: creating script file")
+	try:
+	    sf, scriptname = tempfile.mkstemp()
+	    scriptfile = os.fdopen(sf, 'w')
+	except EnvironmentError:
+	    return False
+
+	scriptbase = os.path.basename(scriptname)
+
+	# Script the filesystem changes
+	print >>scriptfile, "/bin/rm -rf %s" % stagingdir
+	print >>scriptfile, 'mkdir -p %s' % stagingdir
+	print >>scriptfile, "rm -f %s" % scriptbase
+	scriptfile.close()
+
+	# Move the script to the remote machine
+	# XXX: could collide tempfile names on the remote host
+	if segment_commands.scp_file(scriptname, user, host, scriptbase):
+	    os.remove(scriptname)
+	else:
+	    return False
+
+	# Execute the script (and the script's last line deletes it)
+	if not segment_commands.ssh_cmd(user, host, "sh -x %s" % scriptbase):
+	    return False
+
+    def initialize_protogeni_context(self, segment_commands, certfile, certpw):
+	"""
+	Protogeni interactions take a context and a protogeni certificate.
+	This establishes both for later calls and returns them.
+	"""
+	if os.access(certfile, os.R_OK):
+	    ctxt = fedd_ssl_context(my_cert=certfile, password=certpw)
+	else:
+	    self.log.error("[start_segment]: Cannot read certfile: %s" % \
+		    certfile)
+	    return None, None
+
+	try:
+	    gcred = segment_commands.slice_authority_call('GetCredential', 
+		    {}, ctxt)
+	except segment_commands.ProtoGENIError, e:
+	    raise service_error(service_error.federant,
+		    "ProtoGENI: %s" % e)
+
+	return ctxt, gcred
+
+    def get_free_slicename(self, segment_commands, user, gcred, ctxt):
+	"""
+	Find a usable slice name by trying random ones until there's no
+	collision.
+	"""
+
+	def random_slicename(user):
+	    """
+	    Return a random slicename by appending 5 letters to the username.
+	    """
+	    slicename = user
+	    for i in range(0,5):
+		slicename += random.choice(string.ascii_letters)
+	    return slicename
+
+	while True:
+	    slicename = random_slicename(user)
+	    try:
+		param = {
+			'credential': gcred, 
+			'hrn': slicename,
+			'type': 'Slice'
+			}
+		segment_commands.slice_authority_call('Resolve', param, ctxt)
+	    except segment_commands.ProtoGENIError, e:
+		print e
+		break
+
+	return slicename
+
+    def allocate_slice(self, segment_commands, slicename, rspec, gcred, ctxt):
+	"""
+	Create the slice and allocate resources.  If any of this stuff fails,
+	the allocations will time out on PG in short order, so we just raise
+	the service_error.  Return the slice and sliver credentials as well as
+	the manifest.
+	"""
+	try:
+	    param = {
+		    'credential': gcred, 
+		    'hrn': slicename,
+		    'type': 'Slice'
+		    }
+	    slice_cred = segment_commands.slice_authority_call('Register',
+		    param, ctxt)
+	    # Resolve the slice to get the URN that PG has assigned it.
+	    param = {
+		    'credential': gcred,
+		    'type': 'Slice',
+		    'hrn': slicename
+		    }
+	    data = segment_commands.slice_authority_call('Resolve', param,
+		    ctxt)
+	    if 'urn' in data:
+		slice_urn = data['urn']
+	    else:
+		raise service_error(service_error.federant, 
+			"No URN returned for slice %s" % hrn)
+
+	    if 'creator_urn' in data:
+		creator_urn = data['creator_urn']
+	    else:
+		raise service_error(service_error.federant, 
+			"No creator URN returned for slice %s" % hrn)
+	    # Populate the ssh keys (let PG format them)
+	    param = {
+		    'credential': gcred, 
+		    }
+	    keys =  segment_commands.slice_authority_call('GetKeys', param, 
+		    ctxt)
+	    # Create a Sliver
+	    param = {
+		    'credentials': [ slice_cred ],
+		    'rspec': rspec, 
+		    'users': [ {
+			'urn': creator_urn,
+			'keys': keys, 
+			},
+		    ],
+		    'slice_urn': slice_urn,
+		    }
+	    rv = segment_commands.component_manager_call(
+		    'CreateSliver', param, ctxt)
+
+	    # the GENIAPI AM just hands back the manifest, bit the ProtoGENI
+	    # API hands back a sliver credential.  This just finds the format
+	    # of choice.
+	    if isinstance(rv, tuple):
+		manifest = rv[1]
+	    else:
+		manifest = rv
+	except segment_commands.ProtoGENIError, e:
+	    raise service_error(service_error.federant,
+		    "ProtoGENI: %s %s" % (e.code, e))
+
+	return (slice_urn, slice_cred, manifest)
+
+    def wait_for_slice(self, segment_commands, slice_cred, slice_urn, ctxt, 
+	    timeout=None):
+	"""
+	Wait for the given slice to finish its startup.  Return the final
+	status.
+	"""
+	completed_states = ('failed', 'ready')
+	status = 'changing'
+	if timeout is not None:
+	    end = time.time() + timeout
+	try:
+	    while status not in completed_states:
+		param = { 
+			'credentials': [ slice_cred ],
+			'slice_urn': slice_urn,
+			}
+		r = segment_commands.component_manager_call(
+			'SliverStatus', param, ctxt)
+		# GENIAPI uses geni_status as the key, so check for both
+		status = r.get('status', r.get('geni_status','changing'))
+		if status not in completed_states:
+		    if timeout is not None and time.time() > end:
+			return 'timeout'
+		    time.sleep(30)
+	except segment_commands.ProtoGENIError, e:
+	    raise service_error(service_error.federant,
+		    "ProtoGENI: %s %s" % (e.code, e))
+
+	return status
+
+    def delete_slice(self, segment_commands, slice_cred, slice_urn, ctxt):
+	"""
+	Delete the slice resources.  An error from the service is ignores,
+	because the soft state will go away anyway.
+	"""
+	try:
+	    param = { 
+		    'credentials': [ slice_cred, ],
+		    'slice_urn': slice_urn,
+		    }
+	    segment_commands.component_manager_call('DeleteSlice',
+		    param, ctxt)
+	except segment_commands.ProtoGENIError, e:
+	    self.log.warn("ProtoGENI: %s" % e)
+
+
+
+    def start_segment(self, segment_commands, aid, user, rspec, pubkey,
+	    secretkey, ename, stagingdir, tmpdir, certfile, certpw,
+	    export_certfile, topo, connInfo, services, timeout=0):
+	"""
+	Start a sub-experiment on a federant.
+
+	Get the current state, modify or create as appropriate, ship data
+	and configs and start the experiment.  There are small ordering
+	differences based on the initial state of the sub-experiment.
+	"""
+
+	# Local software dir
+	lsoftdir = "%s/software" % tmpdir
+	host = self.staging_host
+
+	ctxt, gcred = self.initialize_protogeni_context(segment_commands, 
+		certfile, certpw)
+
+	if not ctxt: return False
+
+	self.set_up_staging_filespace(segment_commands, user, host, stagingdir)
+	slicename = self.get_free_slicename(segment_commands, user, gcred, ctxt)
+	self.log.info("Creating %s" % slicename)
+	slice_urn, slice_cred, manifest = self.allocate_slice(
+		segment_commands, slicename, rspec, gcred, ctxt)
+
+	# With manifest in hand, we can export the portal node names.
+	if self.create_debug: nodes = self.fake_manifest(topo)
+	else: nodes = self.manifest_to_dict(manifest)
+
+	self.export_store_info(export_certfile, nodes, self.ssh_port,
+		connInfo)
+	self.generate_portal_configs(topo, pubkey, secretkey, tmpdir, 
+		ename, connInfo, services, nodes)
+
+	# Copy software to the staging machine (done after generation to copy
+	# those, too)
+	for d in (tmpdir, lsoftdir):
+	    if os.path.isdir(d):
+		for f in os.listdir(d):
+		    if not os.path.isdir("%s/%s" % (d, f)):
+			if not segment_commands.scp_file("%s/%s" % (d, f), 
+				user, host, "%s/%s" % (stagingdir, f)):
+			    self.log.error("Scp failed")
+			    return False
+
+
+	# Now we wait for the nodes to start on PG
+	status = self.wait_for_slice(segment_commands, slice_cred, slice_urn, 
+		ctxt, timeout=300)
+	if status == 'failed':
+	    self.log.error('Sliver failed to start on ProtoGENI')
+	    self.delete_slice(segment_commands, slice_cred, slice_urn, ctxt)
+	    return False
+	elif status == 'timeout':
+	    self.log.error('Sliver failed to start on ProtoGENI (timeout)')
+	    self.delete_slice(segment_commands, slice_cred, slice_urn, ctxt)
+	    return False
+	else:
+	    # All good: save ProtoGENI info in shared state
+	    self.state_lock.acquire()
+	    self.allocation[aid]['slice_urn'] = slice_urn
+	    self.allocation[aid]['slice_name'] = slicename
+	    self.allocation[aid]['slice_credential'] = slice_cred
+	    self.allocation[aid]['manifest'] = manifest
+	    self.allocation[aid]['certfile'] = certfile
+	    self.allocation[aid]['certpw'] = certpw
+	    self.write_state()
+	    self.state_lock.release()
+
+	# Now we have configuration to do for ProtoGENI
+	self.configure_nodes(segment_commands, topo, nodes, user, pubkey,
+		secretkey, stagingdir, tmpdir)
+
+	self.start_nodes(user, self.staging_host, 
+		[ n.get('hostname', None) for n in nodes.values()],
+		segment_commands)
+
+	# Everything has gone OK.
+	return True, dict([(k, n.get('hostname', None)) \
+		for k, n in nodes.items()])
+
+    def generate_rspec(self, topo, softdir, connInfo):
+
+	# Force a useful image.  Without picking this the nodes can get 
+	# different images and there is great pain.
+	def image_filter(e):
+	    if isinstance(e, topdl.Computer):
+		return '<disk_image name="urn:publicid:IDN+emulab.net+' + \
+			'image+emulab-ops//FEDORA10-STD" />'
+	    else:
+		return ""
+	# Main line of generate
+	t = topo.clone()
+
+	starts = { }
+	# Weed out the things we aren't going to instantiate: Segments, portal
+	# substrates, and portal interfaces.  (The copy in the for loop allows
+	# us to delete from e.elements in side the for loop).  While we're
+	# touching all the elements, we also adjust paths from the original
+	# testbed to local testbed paths and put the federation commands and
+	# startcommands into a dict so we can start them manually later.
+	# ProtoGENI requires setup before the federation commands run, so we
+	# run them by hand after we've seeded configurations.
+	for e in [e for e in t.elements]:
+	    if isinstance(e, topdl.Segment):
+		t.elements.remove(e)
+	    # Fix software paths
+	    for s in getattr(e, 'software', []):
+		s.location = re.sub("^.*/", softdir, s.location)
+	    if isinstance(e, topdl.Computer):
+		if e.get_attribute('portal') and self.portal_startcommand:
+		    # Portals never have a user-specified start command
+		    starts[e.name] = self.portal_startcommand
+		elif self.node_startcommand:
+		    if e.get_attribute('startup'):
+			starts[e.name] = "%s \\$USER '%s'" % \
+				(self.node_startcommand, 
+					e.get_attribute('startup'))
+			e.remove_attribute('startup')
+		    else:
+			starts[e.name] = self.node_startcommand
+
+		# Remove portal interfaces 
+		e.interface = [i for i in e.interface \
+			if not i.get_attribute('portal')]
+
+	t.substrates = [ s.clone() for s in t.substrates ]
+	t.incorporate_elements()
+
+	# Customize the rspec output to use the image we like
+	filters = [ image_filter ]
+
+	# Convert to rspec and return it
+	exp_rspec = topdl.topology_to_rspec(t, filters)
+
+	return exp_rspec
+
+    def retrieve_software(self, topo, certfile, softdir):
+	"""
+	Collect the software that nodes in the topology need loaded and stage
+	it locally.  This implies retrieving it from the experiment_controller
+	and placing it into softdir.  Certfile is used to prove that this node
+	has access to that data (it's the allocation/segment fedid).  Finally
+	local portal and federation software is also copied to the same staging
+	directory for simplicity - all software needed for experiment creation
+	is in softdir.
+	"""
+	sw = set()
+	for e in topo.elements:
+	    for s in getattr(e, 'software', []):
+		sw.add(s.location)
+	os.mkdir(softdir)
+	for s in sw:
+	    self.log.debug("Retrieving %s" % s)
+	    try:
+		get_url(s, certfile, softdir)
+	    except:
+		t, v, st = sys.exc_info()
+		raise service_error(service_error.internal,
+			"Error retrieving %s: %s" % (s, v))
+
+	# Copy local portal node software to the tempdir
+	for s in (self.portal_software, self.federation_software):
+	    for l, f in s:
+		base = os.path.basename(f)
+		copy_file(f, "%s/%s" % (softdir, base))
+
+
+    def initialize_experiment_info(self, attrs, aid, certfile, tmpdir):
+	"""
+	Gather common configuration files, retrieve or create an experiment
+	name and project name, and return the ssh_key filenames.  Create an
+	allocation log bound to the state log variable as well. 
+	"""
+	configs = set(('hosts', 'ssh_pubkey', 'ssh_secretkey'))
+	ename = None
+	pubkey_base = None
+	secretkey_base = None
+	alloc_log = None
+
+	for a in attrs:
+	    if a['attribute'] in configs:
+		try:
+		    self.log.debug("Retrieving %s" % a['value'])
+		    get_url(a['value'], certfile, tmpdir)
+		except:
+		    t, v, st = sys.exc_info()
+		    raise service_error(service_error.internal,
+			    "Error retrieving %s: %s" % (a.get('value', ""), v))
+	    if a['attribute'] == 'ssh_pubkey':
+		pubkey_base = a['value'].rpartition('/')[2]
+	    if a['attribute'] == 'ssh_secretkey':
+		secretkey_base = a['value'].rpartition('/')[2]
+	    if a['attribute'] == 'experiment_name':
+		ename = a['value']
+
+	if not ename:
+	    ename = ""
+	    for i in range(0,5):
+		ename += random.choice(string.ascii_letters)
+	    self.log.warn("No experiment name: picked one randomly: %s" \
+		    % ename)
+
+	self.state_lock.acquire()
+	if self.allocation.has_key(aid):
+	    cf, user, ssh_key, cpw = self.allocation[aid]['credentials']
+	    self.allocation[aid]['experiment'] = ename
+	    self.allocation[aid]['log'] = [ ]
+	    # Create a logger that logs to the experiment's state object as
+	    # well as to the main log file.
+	    alloc_log = logging.getLogger('fedd.access.%s' % ename)
+	    h = logging.StreamHandler(
+		    list_log.list_log(self.allocation[aid]['log']))
+	    # XXX: there should be a global one of these rather than
+	    # repeating the code.
+	    h.setFormatter(logging.Formatter(
+		"%(asctime)s %(name)s %(message)s",
+			'%d %b %y %H:%M:%S'))
+	    alloc_log.addHandler(h)
+	    self.write_state()
+	else:
+	    self.log.error("No allocation for %s!?" % aid)
+	self.state_lock.release()
+
+	return (ename, pubkey_base, secretkey_base, cf, user, ssh_key, 
+		cpw, alloc_log)
+
+    def finalize_experiment(self, topo, nodes, aid, alloc_id):
+	# Copy the assigned names into the return topology
+	rvtopo = topo.clone()
+	embedding = [ ]
+	for k, n in nodes.items():
+	    embedding.append({ 
+		'toponame': k,
+		'physname': [n ],
+		})
+	# Grab the log (this is some anal locking, but better safe than
+	# sorry)
+	self.state_lock.acquire()
+	logv = "".join(self.allocation[aid]['log'])
+	# It's possible that the StartSegment call gets retried (!). 
+	# if the 'started' key is in the allocation, we'll return it rather
+	# than redo the setup.
+	self.allocation[aid]['started'] = { 
+		'allocID': alloc_id,
+		'allocationLog': logv,
+		'segmentdescription': { 
+		    'topdldescription': rvtopo.to_dict() },
+		'embedding': embedding,
+		}
+	retval = copy.deepcopy(self.allocation[aid]['started'])
+	self.write_state()
+	self.state_lock.release()
+
+	return retval
+
+    def StartSegment(self, req, fid):
+	err = None  # Any service_error generated after tmpdir is created
+	rv = None   # Return value from segment creation
+
+	try:
+	    req = req['StartSegmentRequestBody']
+	    topref = req['segmentdescription']['topdldescription']
+	except KeyError:
+	    raise service_error(service_error.req, "Badly formed request")
+
+	connInfo = req.get('connection', [])
+	services = req.get('service', [])
+	auth_attr = req['allocID']['fedid']
+	aid = "%s" % auth_attr
+	attrs = req.get('fedAttr', [])
+	if not self.auth.check_attribute(fid, auth_attr):
+	    raise service_error(service_error.access, "Access denied")
+	else:
+	    # See if this is a replay of an earlier succeeded StartSegment -
+	    # sometimes SSL kills 'em.  If so, replay the response rather than
+	    # redoing the allocation.
+	    self.state_lock.acquire()
+	    retval = self.allocation[aid].get('started', None)
+	    self.state_lock.release()
+	    if retval:
+		self.log.warning("Duplicate StartSegment for %s: " % aid + \
+			"replaying response")
+		return retval
+
+	if topref:
+	    topo = topdl.Topology(**topref)
+	else:
+	    raise service_error(service_error.req, 
+		    "Request missing segmentdescription'")
+
+	certfile = "%s/%s.pem" % (self.certdir, auth_attr)
+        try:
+            tmpdir = tempfile.mkdtemp(prefix="access-")
+	    softdir = "%s/software" % tmpdir
+        except EnvironmentError:
+            raise service_error(service_error.internal, "Cannot create tmp dir")
+
+	# Try block alllows us to clean up temporary files.
+	try:
+	    self.retrieve_software(topo, certfile, softdir)
+	    self.configure_userconf(services, tmpdir)
+	    ename, pubkey_base, secretkey_base, cf, user, ssh_key, \
+		cpw, alloc_log = self.initialize_experiment_info(attrs,
+			aid, certfile, tmpdir)
+	    self.import_store_info(certfile, connInfo)
+	    rspec = self.generate_rspec(topo, "%s/%s/" \
+		    % (self.staging_dir, ename), connInfo)
+
+	    segment_commands = self.api_proxy(keyfile=ssh_key,
+		    debug=self.create_debug, log=alloc_log,
+		    ch_url = self.ch_url, sa_url=self.sa_url,
+		    cm_url=self.cm_url)
+	    rv, nodes = self.start_segment(segment_commands, aid, user, rspec,
+		    pubkey_base, 
+		    secretkey_base, ename,
+		    "%s/%s" % (self.staging_dir, ename), tmpdir, cf, cpw,
+		    certfile, topo, connInfo, services)
+	except EnvironmentError, e:
+	    err = service_error(service_error.internal, "%s" % e)
+	except service_error, e:
+	    err = e
+	except:
+	    t, v, st = sys.exc_info()
+	    err = service_error(service_error.internal, "%s: %s" % \
+		    (v, traceback.extract_tb(st)))
+
+        # Walk up tmpdir, deleting as we go
+	if self.cleanup: self.remove_dirs(tmpdir)
+	else: self.log.debug("[StartSegment]: not removing %s" % tmpdir)
+
+	if rv:
+	    return self.finalize_experiment(topo, nodes, aid, req['allocID'])
+	elif err:
+	    raise service_error(service_error.federant,
+		    "Swapin failed: %s" % err)
+	else:
+	    raise service_error(service_error.federant, "Swapin failed")
+
+    def stop_segment(self, segment_commands, user, stagingdir, slice_cred,
+	    slice_urn, certfile, certpw):
+	"""
+	Stop a sub experiment by calling swapexp on the federant
+	"""
+	host = self.staging_host
+	rv = False
+	try:
+	    # Clean out tar files: we've gone over quota in the past
+	    if stagingdir:
+		segment_commands.ssh_cmd(user, host, "rm -rf %s" % stagingdir)
+	    if slice_cred:
+		self.log.error('Removing Sliver on ProtoGENI')
+		ctxt = fedd_ssl_context(my_cert=certfile, password=certpw)
+		self.delete_slice(segment_commands, slice_cred, slice_urn, ctxt)
+	    return True
+	except self.ssh_cmd_timeout:
+	    rv = False
+	return rv
+
+    def TerminateSegment(self, req, fid):
+	try:
+	    req = req['TerminateSegmentRequestBody']
+	except KeyError:
+	    raise service_error(service_error.req, "Badly formed request")
+
+	auth_attr = req['allocID']['fedid']
+	aid = "%s" % auth_attr
+	attrs = req.get('fedAttr', [])
+	if not self.auth.check_attribute(fid, auth_attr):
+	    raise service_error(service_error.access, "Access denied")
+
+	self.state_lock.acquire()
+	if self.allocation.has_key(aid):
+	    cf, user, ssh_key, cpw = self.allocation[aid]['credentials']
+	    slice_cred = self.allocation[aid].get('slice_credential', None)
+	    slice_urn = self.allocation[aid].get('slice_urn', None)
+	    ename = self.allocation[aid].get('experiment', None)
+	else:
+	    cf, user, ssh_key, cpw = (None, None, None, None)
+	    slice_cred = None
+	    slice_urn = None
+	    ename = None
+	self.state_lock.release()
+
+	if ename:
+	    staging = "%s/%s" % ( self.staging_dir, ename)
+	else:
+	    self.log.warn("Can't find experiment name for %s" % aid)
+	    staging = None
+
+	segment_commands = self.api_proxy(keyfile=ssh_key,
+		debug=self.create_debug, ch_url = self.ch_url,
+		sa_url=self.sa_url, cm_url=self.cm_url)
+	self.stop_segment(segment_commands, user, staging, slice_cred,
+		slice_urn, cf, cpw)
+	return { 'allocID': req['allocID'] }
+
+    def renew_segment(self, segment_commands, name, scred, slice_urn, interval, 
+	    certfile, certpw):
+	"""
+	Linear code through the segment renewal calls.
+	"""
+	ctxt = fedd_ssl_context(my_cert=certfile, password=certpw)
+	try:
+	    expiration = time.strftime("%Y%m%dT%H:%M:%S",
+		    time.gmtime(time.time() + interval))
+	    cred = segment_commands.slice_authority_call('GetCredential', 
+		    {}, ctxt)
+
+	    param = {
+		    'credential': scred,
+		    'expiration': expiration
+		    }
+	    r = segment_commands.slice_authority_call('RenewSlice', param, ctxt)
+	    param = {
+		    'credential': cred,
+		    'hrn': name,
+		    'type': 'Slice',
+		    }
+	    slice = segment_commands.slice_authority_call('Resolve', 
+		    param, ctxt)
+	    uuid = slice.get('uuid', None)
+	    if uuid == None:
+		sys.exit('No uuid for %s' % slicename)
+
+	    print 'Calling GetCredential (uuid)'
+	    param = {
+		    'credential': cred,
+		    'uuid': uuid,
+		    'type': 'Slice',
+		    }
+	    new_scred = segment_commands.slice_authority_call('GetCredential',
+		    param, ctxt)
+
+	except segment_commands.ProtoGENIError, e:
+	    self.log.error("Failed to extend slice %s: %s" % (name, e))
+	    return None
+	try:
+	    print 'Calling RenewSlice (CM)'
+	    param = {
+		    'credentials': [new_scred,],
+		    'slice_urn': slice_urn,
+		    }
+	    r = segment_commands.component_manager_call('RenewSlice', param, 
+		    ctxt)
+	except segment_commands.ProtoGENIError, e:
+	    self.log.warn("Failed to renew sliver for %s: %s" % (name, e))
+
+	return new_scred
+    
+
+    def RenewSlices(self):
+	self.log.info("Scanning for slices to renew")
+	self.state_lock.acquire()
+	aids = self.allocation.keys()
+	self.state_lock.release()
+
+	for aid in aids:
+	    self.state_lock.acquire()
+	    if self.allocation.has_key(aid):
+		name = self.allocation[aid].get('slice_name', None)
+		scred = self.allocation[aid].get('slice_credential', None)
+		slice_urn = self.allocation[aid].get('slice_urn', None)
+		cf, user, ssh_key, cpw = self.allocation[aid]['credentials']
+	    else:
+		name = None
+		scred = None
+	    self.state_lock.release()
+
+	    if not os.access(cf, os.R_OK):
+		self.log.error(
+			"[RenewSlices] cred.file %s unreadable, ignoring" % cf)
+		continue
+
+	    # There's a ProtoGENI slice associated with the segment; renew it.
+	    if name and scred and slice_urn:
+		segment_commands = self.api_proxy(log=self.log, 
+			debug=self.create_debug, keyfile=ssh_key,
+			cm_url = self.cm_url, sa_url = self.sa_url,
+			ch_url = self.ch_url)
+		new_scred = self.renew_segment(segment_commands, name, scred, 
+			slice_urn, self.renewal_interval, cf, cpw)
+		if new_scred:
+		    self.log.info("Slice %s renewed until %s GMT" % \
+			    (name, time.asctime(time.gmtime(\
+				time.time()+self.renewal_interval))))
+		    self.state_lock.acquire()
+		    if self.allocation.has_key(aid):
+			self.allocation[aid]['slice_credential'] = new_scred
+			self.write_state()
+		    self.state_lock.release()
+		else:
+		    self.log.info("Failed to renew slice %s " % name)
+
+	# Let's do this all again soon.  (4 tries before the slices time out)	
+	t = Timer(self.renewal_interval/4, self.RenewSlices)
+	t.start()
Index: fedd/Makefile/federation/protogeni_proxy.py
===================================================================
--- fedd/Makefile/federation/protogeni_proxy.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/protogeni_proxy.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,79 @@
+#!/usr/local/bin/python
+
+import os,sys
+import logging
+
+from service_error import service_error
+from remote_service import xmlrpc_handler, soap_handler, service_caller
+
+from proxy_segment import proxy_segment
+
+
+class protogeni_proxy(proxy_segment):
+    class ProtoGENIError(Exception): 
+	def __init__(self, op, code, output):
+	    Exception.__init__(self, output)
+	    self.op = op
+	    self.code = code
+	    self.output = output
+
+    def __init__(self, log=None, keyfile=None, debug=False, 
+	    ch_url=None, sa_url=None, cm_url=None):
+	proxy_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+
+	self.ProtoGENIError = protogeni_proxy.ProtoGENIError
+	self.ch_url = ch_url
+	self.sa_url = sa_url
+	self.cm_url = cm_url
+
+	self.call_SetValue = service_caller('SetValue')
+
+	self.debug_fail = ['Resolve']
+	self.debug_response = {
+		'RedeemTicket': ("XML blob1", "XML blob2"),
+		'SliceStatus': { 'status': 'ready' },
+	    }
+
+
+    def pg_call(self, url, method, params, context):
+	max_retries = 5
+	retries = 0
+
+	s = service_caller(method, request_body_name="", strict=False)
+	if self.log:
+	    self.log.debug("Calling %s %s" % (url, method))
+	if not self.debug:
+	    while retries < max_retries:
+		r = s.call_xmlrpc_service(url, params, context=context)
+		code = r.get('code', -1)
+		if code == 0:
+		    # Success leaves the loop here
+		    return r.get('value', None)
+		elif code == 14 and retries +1 < max_retries:
+		    # Busy resource
+		    retries+= 1
+		    self.log.info("Resource busy, retrying in 30 secs")
+		    time.sleep(30)
+		else:
+		    # NB: out of retries falls through to here
+		    raise self.ProtoGENIError(op=method, 
+			    code=r.get('code', 'unknown'), 
+			    output=r.get('output', 'No output'))
+	else:
+	    if method in self.debug_fail:
+		raise self.ProtoGENIError(op=method, code='unknown',
+			output='No output')
+	    elif self.debug_response.has_key(method):
+		return self.debug_response[method]
+	    else:
+		return "%s XML blob" % method
+
+    def clearinghouse_call(self, method, params, context):
+	return self.pg_call(self.ch_url, method, params, context)
+
+    def slice_authority_call(self, method, params, context):
+	return self.pg_call(self.sa_url, method, params, context)
+
+    def component_manager_call(self, method, params, context):
+	return self.pg_call(self.cm_url, method, params, context)
+
Index: fedd/Makefile/federation/proxy_emulab_segment.py
===================================================================
--- fedd/Makefile/federation/proxy_emulab_segment.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/proxy_emulab_segment.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,343 @@
+#!/usr/local/bin/python
+
+import sys, os
+import re
+
+import tempfile
+import subprocess
+import logging 
+import time
+import signal
+
+from proxy_segment import proxy_segment
+from service_error import service_error
+
+class start_segment(proxy_segment):
+    """
+    This starts an experiment on an emulab accessed remotely via ssh.  Most of
+    the experiment constuction has been done by the emulab_access object.  This
+    just does the wrangling of the emulab commands and collected the node to
+    physical mapping.  The routine throws service errors.
+    """
+
+    def __init__(self, log=None, keyfile=None, debug=False, boss=None, 
+	    cert=None):
+	proxy_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+	self.null = """
+set ns [new Simulator]
+source tb_compat.tcl
+
+set a [$ns node]
+
+$ns rtproto Session
+$ns run 
+"""
+	self.node = { }
+
+    def get_state(self, user, host, pid, eid):
+	"""
+	Return the state of the experiment as reported by emulab
+	"""
+	# command to test experiment state
+	expinfo_exec = "/usr/testbed/bin/expinfo"  
+	# Regular expressions to parse the expinfo response
+	state_re = re.compile("State:\s+(\w+)")
+	no_exp_re = re.compile("^No\s+such\s+experiment")
+	swapping_re = re.compile("^No\s+information\s+available.")
+	state = None    # Experiment state parsed from expinfo
+	# The expinfo ssh command.  Note the identity restriction to use
+	# only the identity provided in the pubkey given.
+	cmd = [self.ssh_exec, '-o', 'IdentitiesOnly yes', '-o', 
+		'StrictHostKeyChecking no', '-i', 
+		self.ssh_privkey_file, "%s@%s" % (user, host), 
+		expinfo_exec, pid, eid]
+
+	dev_null = None
+	try:
+	    dev_null = open("/dev/null", "a")
+	except EnvironmentError, e:
+	    self.log.error("[get_state]: can't open /dev/null: %s" %e)
+
+	if self.debug:
+	    state = 'swapped'
+	    rv = 0
+	else:
+	    self.log.debug("Checking state")
+	    status = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+		    stderr=dev_null, close_fds=True)
+	    for line in status.stdout:
+		m = state_re.match(line)
+		if m: state = m.group(1)
+		else:
+		    for reg, st in ((no_exp_re, "none"),
+			    (swapping_re, "swapping")):
+			m = reg.match(line)
+			if m: state = st
+	    rv = status.wait()
+
+	# If the experiment is not present the subcommand returns a
+	# non-zero return value.  If we successfully parsed a "none"
+	# outcome, ignore the return code.
+	if rv != 0 and state != 'none':
+	    raise service_error(service_error.internal,
+		    "Cannot get status of segment:%s/%s" % (pid, eid))
+	elif state not in ('active', 'swapped', 'swapping', 'none'):
+	    raise service_error(service_error.internal,
+		    "Cannot get status of segment:%s/%s" % (pid, eid))
+	else:
+	    self.log.debug("State is %s" % state)
+	    return state
+
+
+    def get_mapping(self, user, host, pid, eid):
+	"""
+	Get the physical to virtual mapping from the expinfo command and save
+	it in the self.map member.
+	"""
+	# command to test experiment state
+	expinfo_exec = "/usr/testbed/bin/expinfo"  
+	# The expinfo ssh command.  Note the identity restriction to use
+	# only the identity provided in the pubkey given.
+	cmd = [self.ssh_exec, '-o', 'IdentitiesOnly yes', '-o', 
+		'StrictHostKeyChecking no', '-i', 
+		self.ssh_privkey_file, "%s@%s" % (user, host), 
+		expinfo_exec, '-m', pid, eid]
+
+	dev_null = None
+	try:
+	    dev_null = open("/dev/null", "a")
+	except EnvironmentError, e:
+	    self.log.error("[get_mapping]: can't open /dev/null: %s" %e)
+
+	if self.debug:
+	    rv = 0
+	else:
+	    self.log.debug("Getting mapping for %s %s" % (pid, eid))
+	    phys_start = re.compile('^Physical\s+Node\s+Mapping')
+	    phys_line = re.compile('(\S+)(\s+\S+)*\s+(\S+)')
+	    phys_end = re.compile('^$')
+	    status = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+		    stderr=dev_null, close_fds=True)
+
+	    # Parse the info output.  Format:
+	    #
+	    # stuff
+	    # Physical Node Mapping:
+	    # ID              Type         OS              Physical    
+	    # --------------- ------------ --------------- ------------
+	    # virtual         dummy        dummy           physical
+	    #
+	    foundit = False
+	    skip = 0
+	    for line in status.stdout:
+		if phys_start.match(line):
+		    skip = 2
+		    foundit = True
+		elif not foundit:
+		    continue
+		elif skip > 0:
+		    skip -= 1
+		elif phys_end.match(line):
+		    break
+		else:
+		    m = phys_line.match(line.strip())
+		    if m: self.node[m.group(1)] = m.group(3)
+		    else: self.log.warn(
+			    "Matching failed while parsing node mapping: " +\
+				    "line %s" % line)
+	    rv = status.wait()
+
+	# If the experiment is not present the subcommand returns a
+	# non-zero return value.  If we successfully parsed a "none"
+	# outcome, ignore the return code.
+	if rv != 0 :
+	    raise service_error(service_error.internal,
+		    "Cannot get node mapping of segment:%s/%s" % (pid, eid))
+	else:
+	    return True
+
+
+    def make_null_experiment(self, user, host, pid, eid, tmpdir):
+	"""
+	Create a null copy of the experiment so that we capture any logs there
+	if the modify fails.  Emulab software discards the logs from a failed
+	startexp
+	"""
+	try:
+	    f = open("%s/null.tcl" % tmpdir, "w")
+	    print >>f, self.null
+	    f.close()
+	except EnvironmentError, e:
+	    raise service_error(service_error.internal,
+		    "Cannot stage tarfile/rpm: %s" % e.strerror)
+
+	if not self.scp_file("%s/null.tcl" % tmpdir, user, host):
+	    return False
+	self.log.info("[start_segment]: Creating %s" % eid)
+	timedout = False
+	try:
+	    if not self.ssh_cmd(user, host,
+		    ("/usr/testbed/bin/startexp -i -f -w -p %s " + 
+		    "-e %s null.tcl") % (pid, eid), "startexp",
+		    timeout=60 * 10):
+		return False
+	except self.ssh_cmd_timeout:
+	    timedout = True
+
+	if timedout:
+	    state = self.get_state(user, host, pid, eid)
+	    if state != "swapped":
+		return False
+	return True
+
+    def set_up_experiment_filespace(self, user, host, pid, eid, tmpdir):
+	"""
+	Send all the software and configuration files into the experiment's
+	file space.  To reduce the number of ssh connections, we script many
+	changes and execute the script.
+	"""
+	# Configuration directories on the remote machine
+	proj_dir = "/proj/%s/exp/%s/tmp" % (pid, eid)
+	softdir = "/proj/%s/software/%s" % (pid, eid)
+	# Local software dir
+	lsoftdir = "%s/software" % tmpdir
+
+	# Open up a temporary file to contain a script for setting up the
+	# filespace for the new experiment.
+	self.log.info("[start_segment]: creating script file")
+	try:
+	    sf, scriptname = tempfile.mkstemp()
+	    scriptfile = os.fdopen(sf, 'w')
+	except EnvironmentError:
+	    return False
+
+	scriptbase = os.path.basename(scriptname)
+
+	# Script the filesystem changes
+	print >>scriptfile, "/bin/rm -rf %s" % proj_dir
+	# Clear and create the software directory
+	print >>scriptfile, "/bin/rm -rf %s/*" % softdir
+	print >>scriptfile, 'mkdir -p %s' % proj_dir
+	if os.path.isdir(lsoftdir):
+	    print >>scriptfile, 'mkdir -p %s' % softdir
+	print >>scriptfile, "rm -f %s" % scriptbase
+	scriptfile.close()
+
+	# Move the script to the remote machine
+	# XXX: could collide tempfile names on the remote host
+	if self.scp_file(scriptname, user, host, scriptbase):
+	    os.remove(scriptname)
+	else:
+	    return False
+
+	# Execute the script (and the script's last line deletes it)
+	if not self.ssh_cmd(user, host, "sh -x %s" % scriptbase):
+	    return False
+
+	for f in os.listdir(tmpdir):
+	    if not os.path.isdir("%s/%s" % (tmpdir, f)):
+		if not self.scp_file("%s/%s" % (tmpdir, f), user, host,
+			"%s/%s" % (proj_dir, f)):
+		    return False
+	if os.path.isdir(lsoftdir):
+	    for f in os.listdir(lsoftdir):
+		if not os.path.isdir("%s/%s" % (lsoftdir, f)):
+		    if not self.scp_file("%s/%s" % (lsoftdir, f), 
+			    user, host, "%s/%s" % (softdir, f)):
+			return False
+	return True
+
+    def swap_in(self, user, host, pid, eid):
+	"""
+	Swap experiment in.  This includes code to cope with the experiment
+	swaping command timing out, but the experiment being swapped in
+	successfully.
+	"""
+	self.log.info("[start_segment]: Swapping %s in" % eid)
+	timedout = False
+	try:
+	    if not self.ssh_cmd(user, host,
+		    "/usr/testbed/bin/swapexp -w %s %s in" % (pid, eid),
+		    "swapexp", timeout=25*60):
+		return False
+	except self.ssh_cmd_timeout:
+	    timedout = True
+       
+	# If the command was terminated, but completed successfully,
+	# report success.
+	if timedout:
+	    self.log.debug("[start_segment]: swapin timed out " +\
+		    "checking state")
+	    state = self.get_state(user, host, pid, eid)
+	    self.log.debug("[start_segment]: state is %s" % state)
+	    return state == 'active'
+
+	return True
+
+
+    def __call__(self, parent, eid, pid, user, tclfile, tmpdir, timeout=0):
+	"""
+	Start a sub-experiment on a federant.
+
+	Get the current state, modify or create as appropriate, ship data
+	and configs and start the experiment.  There are small ordering
+	differences based on the initial state of the sub-experiment.
+	"""
+	# ops node in the federant
+	host = "%s%s" % (parent.ops, parent.domain)
+	state = self.get_state(user, host, pid, eid)
+
+	if not self.scp_file(tclfile, user, host):
+	    return False
+	
+	if state == 'none':
+	    # Put a dummy in place to capture logs, and establish an experiment
+	    # directory.
+	    if not self.make_null_experiment(user, host, pid, eid, tmpdir):
+		return False
+
+	if not self.set_up_experiment_filespace(user, host, pid, eid, tmpdir):
+	    return False
+
+	# With the filespace in place, we can modify and swap in.
+	self.log.info("[start_segment]: Modifying %s" % eid)
+	try:
+	    if not self.ssh_cmd(user, host,
+		    "/usr/testbed/bin/modexp -r -s -w %s %s %s" % \
+			    (pid, eid, tclfile.rpartition('/')[2]),
+		    "modexp", timeout= 60 * 10):
+		return False
+	except self.ssh_cmd_timeout:
+	    self.log.error("Modify command failed to complete in time")
+	    # There's really no way to see if this succeeded or failed, so
+	    # if it hangs, assume the worst.
+	    return False
+	# Active experiments are still swapped, this swaps the others in.
+	if state != 'active':
+	    if not self.swap_in(user, host, pid, eid):
+		return False
+	# Everything has gone OK.
+	self.get_mapping(user, host, pid,eid)
+	return True
+
+class stop_segment(proxy_segment):
+    def __init__(self, log=None, keyfile=None, debug=False, boss=None, cert=None):
+	proxy_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+
+    def __call__(self, parent, user, pid, eid):
+	"""
+	Stop a sub experiment by calling swapexp on the federant
+	"""
+	host = "%s%s" % (parent.ops, parent.domain)
+	self.log.info("[stop_segment]: Stopping %s" % eid)
+	rv = False
+	try:
+	    # Clean out tar files: we've gone over quota in the past
+	    self.ssh_cmd(user, host, "rm -rf /proj/%s/software/%s" % \
+		    (pid, eid))
+	    rv = self.ssh_cmd(user, host,
+		    "/usr/testbed/bin/swapexp -w %s %s out" % (pid, eid))
+	except self.ssh_cmd_timeout:
+	    rv = False
+	return rv
+
Index: fedd/Makefile/federation/proxy_segment.py
===================================================================
--- fedd/Makefile/federation/proxy_segment.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/proxy_segment.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,109 @@
+#!/usr/local/bin/python
+
+import sys, os
+import re
+
+import tempfile
+import subprocess
+import logging 
+import time
+import signal
+
+from service_error import service_error
+
+class proxy_segment:
+    """
+    Base class for segment starter classes that access their underlying testebd
+    remotely using ssh.  It is promarily a code repository for commonly used
+    ssh commands for moving code and logging in.
+    """
+    class ssh_cmd_timeout(RuntimeError): pass
+
+    def __init__(self, log=None, keyfile=None, debug=False):
+	"""
+	log is the logging.log to print messages to, keyfile is the private key
+	for ssh interactions and if debug is true, commands are not executed
+	using ssh.
+	"""
+	self.log = log or logging.getLogger('fedd.access.proxy_segment')
+	self.ssh_privkey_file = keyfile
+	self.debug = debug
+	self.ssh_exec="/usr/bin/ssh"
+	self.scp_exec = "/usr/bin/scp"
+	self.ssh_cmd_timeout = proxy_segment.ssh_cmd_timeout
+
+    def scp_file(self, file, user, host, dest=""):
+	"""
+	scp a file to the remote host.  If debug is set the action is only
+	logged.
+	"""
+
+	scp_cmd = [self.scp_exec, '-o', 'IdentitiesOnly yes', 
+		'-o', 'StrictHostKeyChecking no', '-o', 'ForwardX11 no','-i', 
+		self.ssh_privkey_file, file, 
+		"%s@%s:%s" % (user, host, dest)]
+	rv = 0
+
+	try:
+	    dnull = open("/dev/null", "w")
+	except EnvironmentError:
+	    self.log.debug("[ssh_file]: failed to open " + \
+		    "/dev/null for redirect")
+	    dnull = Null
+
+	self.log.debug("[scp_file]: %s" % " ".join(scp_cmd))
+	if not self.debug:
+	    rv = subprocess.call(scp_cmd, stdout=dnull, 
+		    stderr=dnull, close_fds=True)
+
+	return rv == 0
+
+    def ssh_cmd(self, user, host, cmd, wname=None, timeout=None):
+	"""
+	Run a remote command on host as user.  If debug is set, the action
+	is only logged.  Commands are run without stdin, to avoid stray
+	SIGTTINs.
+	"""
+	sh_str = ("%s -n -o 'IdentitiesOnly yes' -o " + \
+		"'StrictHostKeyChecking no' -o 'ForwardX11 no' " +
+		"-i %s %s@%s %s") % \
+		(self.ssh_exec, self.ssh_privkey_file, 
+			user, host, cmd)
+
+	try:
+	    dnull = open("/dev/null", "w")
+	except EnvironmentError:
+	    self.log.debug("[ssh_cmd]: failed to open /dev/null " + \
+		    "for redirect")
+	    dnull = Null
+
+	self.log.debug("[ssh_cmd]: %s" % sh_str)
+	if not self.debug:
+	    if dnull:
+		sub = subprocess.Popen(sh_str, shell=True, stdout=dnull,
+			stderr=dnull, close_fds=True)
+	    else:
+		sub = subprocess.Popen(sh_str, shell=True, close_fds=True)
+	    if timeout:
+		i = 0
+		rv = sub.poll()
+		while i < timeout:
+		    if rv is not None: break
+		    else:
+			time.sleep(1)
+			rv = sub.poll()
+			i += 1
+		else:
+		    self.log.debug("Process exceeded runtime: %s" % sh_str)
+		    os.kill(sub.pid, signal.SIGKILL)
+		    raise self.ssh_cmd_timeout();
+		return rv == 0
+	    else:
+		return sub.wait() == 0
+	else:
+	    if timeout == 0:
+		self.log.debug("debug timeout raised on %s " % sh_str)
+		raise self.ssh_cmd_timeout()
+	    else:
+		return True
+
Index: fedd/Makefile/federation/remote_service.py
===================================================================
--- fedd/Makefile/federation/remote_service.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/remote_service.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,550 @@
+#!/usr/local/bin/python
+
+import copy
+
+from socket import error as socket_error
+from socket import sslerror
+
+import M2Crypto.httpslib
+from M2Crypto import SSL
+from M2Crypto.m2xmlrpclib import SSL_Transport
+from M2Crypto.SSL import SSLError
+from M2Crypto.BIO import BIOError
+from ZSI import ParseException, FaultException, SoapWriter
+
+# Underlying SOAP comms use this and we need to catch their exceptions
+import httplib
+
+from service_error import service_error
+from xmlrpclib import ServerProxy, dumps, loads, Fault, Error, Binary
+
+import fedd_services
+import fedd_internal_services
+from util import fedd_ssl_context
+from fedid import fedid
+import parse_detail
+
+# Turn off the matching of hostname to certificate ID
+SSL.Connection.clientPostConnectionCheck = None
+
+# Used by the remote_service_base class.
+def to_binary(o):
+    """
+    A function that converts an object into an xmlrpclib.Binary using
+    either its internal packing method, or the standard Binary constructor.
+    """
+    pack = getattr(o, 'pack_xmlrpc', None)
+    if callable(pack): return Binary(pack())
+    else: return Binary(o)
+
+# Classes that encapsulate the process of making and dealing with requests to
+# WSDL-generated and XMLRPC remote accesses.  
+
+class remote_service_base:
+    """
+    This invisible base class encapsulates the functions used to massage the
+    dictionaries used to pass parameters into and out of the RPC formats.  It's
+    mostly a container for the static methods to do that work, but defines some
+    maps sued by sub classes on apply_to_tags
+    """
+    # A map used to convert fedid fields to fedid objects (when the field is
+    # already a string)
+    fedid_to_object = ( ('fedid', lambda x: fedid(bits=x)),)
+    # A map used by apply_to_tags to convert fedids from xmlrpclib.Binary
+    # objects to fedid objects in one sweep.
+    decap_fedids = (('fedid', lambda x: fedid(bits=x.data)),)
+    # A map used to encapsulate fedids into xmlrpclib.Binary objects
+    encap_fedids = (('fedid', to_binary),)
+
+    @staticmethod
+    def pack_soap(container, name, contents):
+	"""
+	Convert the dictionary in contents into a tree of ZSI classes.
+
+	The holder classes are constructed from factories in container and
+	assigned to either the element or attribute name.  This is used to
+	recursively create the SOAP message.
+	"""
+	if getattr(contents, "__iter__", None) != None:
+	    attr =getattr(container, "new_%s" % name, None)
+	    if attr: obj = attr()
+	    else:
+		raise TypeError("%s does not have a new_%s attribute" % \
+			(container, name))
+	    for e, v in contents.iteritems():
+		assign = getattr(obj, "set_element_%s" % e, None) or \
+			getattr(obj, "set_attribute_%s" % e, None)
+		if isinstance(v, type(dict())):
+		    assign(remote_service_base.pack_soap(obj, e, v))
+		elif getattr(v, "__iter__", None) != None:
+		    assign([ remote_service_base.pack_soap(obj, e, val ) \
+			    for val in v])
+		elif getattr(v, "pack_soap", None) != None:
+		    assign(v.pack_soap())
+		else:
+		    assign(v)
+	    return obj
+	else: return contents
+
+    @staticmethod
+    def unpack_soap(element):
+	"""
+	Convert a tree of ZSI SOAP classes intro a hash.  The inverse of
+	pack_soap
+
+	Elements or elements that are empty are ignored.
+	"""
+	methods = [ m for m in dir(element) \
+		if m.startswith("get_element") or m.startswith("get_attribute")]
+	if len(methods) > 0:
+	    rv = { }
+	    for m in methods:
+		if m.startswith("get_element_"):
+		    n = m.replace("get_element_","",1)
+		else:
+		    n = m.replace("get_attribute_", "", 1)
+		sub = getattr(element, m)()
+		if sub != None:
+		    if isinstance(sub, basestring):
+			rv[n] = sub
+		    elif getattr(sub, "__iter__", None) != None:
+			if len(sub) > 0: rv[n] = \
+				[remote_service_base.unpack_soap(e) \
+				    for e in sub]
+		    else:
+			rv[n] = remote_service_base.unpack_soap(sub)
+	    return rv
+	else: 
+	    return element
+
+    @staticmethod
+    def apply_to_tags(e, map):
+	"""
+	Map is an iterable of ordered pairs (tuples) that map a key to a
+	function.
+	This function walks the given message and replaces any object with a
+	key in the map with the result of applying that function to the object.
+	"""
+	if isinstance(e, dict):
+	    for k in e.keys():
+		for tag, fcn in map:
+		    if k == tag:
+			if isinstance(e[k], list):
+			    e[k] = [ fcn(b) for b in e[k]]
+			else:
+			    e[k] = fcn(e[k])
+		    elif isinstance(e[k], dict):
+			remote_service_base.apply_to_tags(e[k], map)
+		    elif isinstance(e[k], list):
+			for ee in e[k]:
+			    remote_service_base.apply_to_tags(ee, map)
+	# Other types end the recursion - they should be leaves
+	return e
+
+    @staticmethod
+    def strip_unicode(obj):
+	"""Walk through a message and convert all strings to non-unicode
+	strings"""
+	if isinstance(obj, dict):
+	    for k in obj.keys():
+		obj[k] = remote_service_base.strip_unicode(obj[k])
+	    return obj
+	elif isinstance(obj, basestring) and not isinstance(obj, str):
+	    return str(obj)
+	elif getattr(obj, "__iter__", None):
+	    return [ remote_service_base.strip_unicode(x) for x in obj]
+	else:
+	    return obj
+
+    @staticmethod
+    def make_unicode(obj):
+	"""Walk through a message and convert all strings to unicode"""
+	if isinstance(obj, dict):
+	    for k in obj.keys():
+		obj[k] = remote_service_base.make_unicode(obj[k])
+	    return obj
+	elif isinstance(obj, basestring) and not isinstance(obj, unicode):
+	    return unicode(obj)
+	elif getattr(obj, "__iter__", None):
+	    return [ remote_service_base.make_unicode(x) for x in obj]
+	else:
+	    return obj
+
+
+
+class soap_handler(remote_service_base):
+    """
+    Encapsulate the handler code to unpack and pack SOAP requests and responses
+    and call the given method.
+
+    The code to decapsulate and encapsulate parameters encoded in SOAP is the
+    same modulo a few parameters.  This is a functor that calls a fedd service
+    trhough a soap interface.  The parameters are the typecode of the request
+    parameters, the method to call (usually a bound instance of a method on a
+    fedd service providing class), the constructor of a response packet and the
+    name of the body element of that packet.  The handler takes a ParsedSoap
+    object (the request) and returns an instance of the class created by
+    constructor containing the response.  Failures of the constructor or badly
+    created constructors will result in None being returned.
+    """
+    def __init__(self, service_name, method, typecode=None,
+	    constructor=None, body_name=None):
+	self.method = method
+
+	response_class_name = "%sResponseMessage" % service_name
+	request_class_name = "%sRequestMessage" % service_name
+
+	if body_name: self.body_name = body_name
+	else: self.body_name = "%sResponseBody" % service_name
+
+	if constructor: self.constructor = constructor
+	else:
+	    self.constructor = self.get_class(response_class_name)
+	    if not self.constructor:
+		raise service_error(service_error.internal,
+			"Cannot find class for %s" % response_class_name)
+
+	if typecode: self.typecode = typecode
+	else: 
+	    req = self.get_class(request_class_name)
+	    if req:
+		self.typecode = req.typecode
+	    else:
+		raise service_error(service_error.internal,
+			"Cannot find class for %s" % request_class_name)
+
+	    if not self.typecode:
+		raise service_error(service_error.internal,
+			"Cannot get typecode for %s" % class_name)
+
+    def get_class(self, class_name):
+	return getattr(fedd_services, class_name, None) or \
+		getattr(fedd_internal_services, class_name, None)
+
+    def __call__(self, ps, fid):
+	req = ps.Parse(self.typecode)
+	# Convert the message to a dict with the fedid strings converted to
+	# fedid objects
+	req = self.apply_to_tags(self.unpack_soap(req), self.fedid_to_object)
+
+	msg = self.method(req, fid)
+
+	resp = self.constructor()
+	set_element = getattr(resp, "set_element_%s" % self.body_name, None)
+	if set_element and callable(set_element):
+	    try:
+		set_element(self.pack_soap(resp, self.body_name, msg))
+		return resp
+	    except (NameError, TypeError):
+		return None
+	else:
+	    return None
+
+class xmlrpc_handler(remote_service_base):
+    """
+    Generate the handler code to unpack and pack XMLRPC requests and responses
+    and call the given method.
+
+    The code to marshall and unmarshall XMLRPC parameters to and from a fedd
+    service is largely the same.  This helper creates such a handler.  The
+    parameters are the method name, and the name of the body struct that
+    contains the response.  A handler is created that takes the params response
+    from an xmlrpclib.loads on the incoming rpc and a fedid and responds with
+    a hash representing the struct ro be returned to the other side.  On error
+    None is returned.  Fedid fields are decapsulated from binary and converted
+    to fedid objects on input and encapsulated as Binaries on output.
+    """
+    def __init__(self, service_name, method):
+	self.method = method
+	self.body_name = "%sResponseBody" % service_name
+
+    def __call__(self, params, fid):
+	msg = None
+
+	p = self.apply_to_tags(params[0], self.decap_fedids)
+	try:
+	    msg = self.method(p, fid)
+	except service_error, e:
+	    raise Fault(e.code, "%s: %s" % (e.code_string(), e.desc))
+	if msg != None:
+	    return self.make_unicode(self.apply_to_tags(\
+		    { self.body_name: msg }, self.encap_fedids))
+	else:
+	    return None
+
+class service_caller(remote_service_base):
+    def __init__(self, service_name, request_message=None, 
+	    request_body_name=None, tracefile=None, strict=True,
+	    log=None, max_retries=None):
+	self.service_name = service_name
+
+	if getattr(fedd_services.feddBindingSOAP, service_name, None):
+	    self.locator = fedd_services.feddServiceLocator
+	    self.port_name = 'getfeddPortType'
+	elif getattr(fedd_internal_services.feddInternalBindingSOAP, 
+		service_name, None):
+	    self.locator = fedd_internal_services.feddInternalServiceLocator
+	    self.port_name = 'getfeddInternalPortType'
+
+	if request_message: self.request_message = request_message
+	else:
+	    request_message_name = "%sRequestMessage" % service_name
+	    self.request_message = \
+		    getattr(fedd_services, request_message_name, None) or \
+		    getattr(fedd_internal_services, request_message_name, None)
+	    if not self.request_message and strict:
+		raise service_error(service_error.internal,
+			"Cannot find class for %s" % request_message_name)
+
+	if request_body_name is not None:
+	    self.request_body_name = request_body_name
+	else: 
+	    self.request_body_name = "%sRequestBody" % service_name
+
+	self.tracefile = tracefile
+	self.__call__ = self.call_service
+	if max_retries is not None: self.max_retries = max_retries 
+	else: self.max_retries = 5
+	self.log = log
+
+    def serialize_soap(self, req):
+	"""
+	Return a string containing the message that would be sent to call this
+	service with the given request.
+	"""
+	msg = self.request_message()
+	set_element = getattr(msg, "set_element_%s" % self.request_body_name,
+		None)
+	if not set_element:
+	    raise service_error(service_error.internal,
+		    "Cannot get element setting method for %s" % \
+			    self.request_body_name)
+	set_element(self.pack_soap(msg, self.request_body_name, req))
+	sw = SoapWriter()
+	sw.serialize(msg)
+	return unicode(sw)
+
+    def call_xmlrpc_service(self, url, req, cert_file=None, cert_pwd=None, 
+	    trusted_certs=None, context=None, tracefile=None):
+	"""Send an XMLRPC request.  """
+
+
+	# If a context is given, use it.  Otherwise construct one from
+	# components.  The construction shouldn't call out for passwords.
+	if context:
+	    ctx = context
+	else:
+	    try:
+		ctx = fedd_ssl_context(cert_file, trusted_certs, 
+			password=cert_pwd)
+	    except SSL.SSLError:
+		raise service_error(service_error.server_config,
+			"Certificates misconfigured")
+
+	# Of all the dumbass things.  The XMLRPC library in use here won't
+	# properly encode unicode strings, so we make a copy of req with
+	# the unicode objects converted.  We also convert the url to a
+	# basic string if it isn't one already.
+	r = self.strip_unicode(copy.deepcopy(req))
+	if self.request_body_name:
+	    r  = self.apply_to_tags(\
+		    { self.request_body_name: r}, self.encap_fedids)
+	else:
+	    r = self.apply_to_tags(r, self.encap_fedids)
+
+	url = str(url)
+	ok = False
+	retries = 0
+
+	while not ok and retries < self.max_retries:
+	    try:
+		transport = SSL_Transport(ctx)
+		port = ServerProxy(url, transport=transport)
+		remote_method = getattr(port, self.service_name, None)
+		resp = remote_method(r)
+		ok = True
+	    except socket_error, e:
+		raise service_error(service_error.connect, 
+			"Cannot connect to %s: %s" % (url, e[1]))
+	    except BIOError, e:
+		if self.log:
+		    self.log.warn("BIO error contacting %s: %s" % (url, e))
+		retries += 1
+	    except sslerror, e:
+		if self.log:
+		    self.log.warn("SSL (socket) error contacting %s: %s" % 
+			    (url, e))
+		retries += 1
+	    except SSLError, e:
+		if self.log:
+		    self.log.warn("SSL error contacting %s: %s" % (url, e))
+		retries += 1
+	    except httplib.HTTPException, e:
+		if self.log:
+		    self.log.warn("HTTP error contacting %s: %s" % (url, e))
+		retries +=1
+	    except Fault, f:
+		raise service_error(f.faultCode, f.faultString)
+	    except Error, e:
+		raise service_error(service_error.protocol, 
+			"Remote XMLRPC Fault: %s" % e)
+
+	if retries >= self.max_retries :
+	    raise service_error(service_error.connect, "Too many SSL failures")
+
+	return self.apply_to_tags(resp, self.decap_fedids) 
+
+    def call_soap_service(self, url, req, cert_file=None, cert_pwd=None,
+	    trusted_certs=None, context=None, tracefile=None):
+	"""
+	Send req on to the real destination in dt and return the response
+
+	Req is just the requestType object.  This function re-wraps it.  It
+	also rethrows any faults.
+	"""
+
+	tf = tracefile or self.tracefile or None
+
+	if not self.request_body_name:
+	    raise service_error(service_error.internal, 
+		    "Call to soap service without a configured request body");
+
+	ok = False
+	retries = 0
+	while not ok and retries < self.max_retries:
+	    try:
+		# Reconstruct the full request message
+		msg = self.request_message()
+		set_element = getattr(msg, "set_element_%s" % \
+			self.request_body_name,
+			None)
+		if not set_element:
+		    raise service_error(service_error.internal,
+			    "Cannot get element setting method for %s" % \
+				    self.request_body_name)
+		set_element(self.pack_soap(msg, self.request_body_name, req))
+		# If a context is given, use it.  Otherwise construct one from
+		# components.  The construction shouldn't call out for
+		# passwords.
+		if context:
+		    if self.log:
+			self.log.debug("Context passed in to call_soap")
+		    ctx = context
+		else:
+		    if self.log:
+			self.log.debug(
+				"Constructing context in call_soap: %s" % \
+					cert_file)
+		    try:
+			ctx = fedd_ssl_context(cert_file, trusted_certs, 
+				password=cert_pwd)
+		    except SSL.SSLError, e:
+			if self.log:
+			    self.log.debug("Certificate error: %s" % e)
+			raise service_error(service_error.server_config,
+				"Certificates misconfigured")
+		loc = self.locator()
+		get_port = getattr(loc, self.port_name, None)
+		if not get_port:
+		    raise service_error(service_error.internal, 
+			    "Cannot get port %s from locator" % self.port_name)
+		port = get_port(url,
+			transport=M2Crypto.httpslib.HTTPSConnection, 
+			transdict={ 'ssl_context' : ctx },
+			tracefile=tf)
+		remote_method = getattr(port, self.service_name, None)
+		if not remote_method:
+		    raise service_error(service_error.internal,
+			    "Cannot get service from SOAP port")
+
+		fail_exc = None
+		if self.log:
+		    self.log.debug("Calling %s (retry %d)" % \
+			    (self.service_name, retries))
+		resp = remote_method(msg)
+		ok = True
+	    except socket_error, e:
+		raise service_error(service_error.connect, 
+			"Cannot connect to %s: %s" % (url, e[1]))
+	    except BIOError, e:
+		if self.log:
+		    self.log.warn("BIO error contacting %s: %s" % (url, e))
+		fail_exc = e
+		retries += 1
+	    except sslerror, e:
+		if self.log:
+		    self.log.warn("SSL (socket) error contacting %s: %s" % 
+			    (url, e))
+		retries += 1
+	    except SSLError, e:
+		if self.log:
+		    self.log.warn("SSL error contacting %s: %s" % (url, e))
+		fail_exc = e
+		retries += 1
+	    except httplib.HTTPException, e:
+		if self.log:
+		    self.log.warn("HTTP error contacting %s: %s" % (url, e))
+		fail_exc = e
+		retries +=1
+	    except ParseException, e:
+		raise service_error(service_error.protocol,
+			"Bad format message (XMLRPC??): %s" % e)
+	    except FaultException, e:
+		# If the method isn't implemented we get a FaultException
+		# without a detail (which would be a FeddFault).  If that's the
+		# case construct a service_error out of the SOAP fields of the
+		# fault, if they're present.
+		if e.fault.detail:
+		    det = e.fault.detail[0]
+		    ee = self.unpack_soap(det).get('FeddFaultBody', { })
+		else:
+		    ee = { 'code': service_error.internal, 
+			    'desc': e.fault.string or "Something Weird" }
+		if ee:
+		    raise service_error(ee['code'], ee['desc'])
+		else:
+		    raise service_error(service_error.internal,
+			    "Unexpected fault body")
+
+	if retries >= self.max_retries and fail_exc and not ok:
+	    raise service_error(service_error.connect, 
+		    "Too many failures: %s" % fail_exc)
+
+	# Unpack and convert fedids to objects
+	r = self.apply_to_tags(self.unpack_soap(resp), self.fedid_to_object)
+
+	#  Make sure all strings are unicode
+	r = self.make_unicode(r)
+	return r
+
+    def call_service(self, url, req, cert_file=None, cert_pwd=None, 
+	trusted_certs=None, context=None, tracefile=None):
+	p_fault = None	# Any SOAP failure (sent unless XMLRPC works)
+	resp = None
+	try:
+	    # Try the SOAP request
+	    resp = self.call_soap_service(url, req, 
+		    cert_file, cert_pwd, trusted_certs, context, tracefile)
+	    return resp
+	except service_error, e:
+	    if e.code == service_error.protocol: p_fault = None
+	    else: raise
+	except FaultException, f:
+	    p_fault = f.fault.detail[0]
+		
+
+	# If we could not get a valid SOAP response to the request above,
+	# try the same address using XMLRPC and let any faults flow back
+	# out.
+	if p_fault == None:
+	    resp = self.call_xmlrpc_service(url, req, cert_file,
+		    cert_pwd, trusted_certs, context, tracefile)
+	    return resp
+	else:
+	    # Build the fault
+	    ee = unpack_soap(p_fault).get('FeddFaultBody', { })
+	    if ee:
+		raise service_error(ee['code'], ee['desc'])
+	    else:
+		raise service_error(service_error.internal,
+			"Unexpected fault body")
Index: fedd/Makefile/federation/server.py
===================================================================
--- fedd/Makefile/federation/server.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/server.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,271 @@
+#!/usr/local/bin/python
+
+import sys
+
+from BaseHTTPServer import BaseHTTPRequestHandler
+
+from ZSI import Fault, ParseException, FaultFromNotUnderstood, \
+    FaultFromZSIException, FaultFromException, ParsedSoap, SoapWriter
+
+from M2Crypto import SSL
+from M2Crypto.SSL.SSLServer import ThreadingSSLServer
+import xmlrpclib
+
+from fedid import fedid
+from fedd_services import ns0
+from service_error import *
+
+import os.path
+
+import logging
+import traceback
+
+# The SSL server here is based on the implementation described at
+# http://www.xml.com/pub/a/ws/2004/01/20/salz.html
+
+# Turn off the matching of hostname to certificate ID
+SSL.Connection.clientPostConnectionCheck = None
+
+class server(ThreadingSSLServer):
+    """
+    Interface the fedd services to the XMLRPC and SOAP interfaces
+    """
+    def __init__(self, ME, handler, ssl_ctx, impl, debug=False):
+	"""
+	Create an SSL server that handles the transport in handler using the
+	credentials in ssl_ctx, and interfacing to the implementation of fedd
+	services in fedd.  ME is the host port pair on which to bind.
+	"""
+	ThreadingSSLServer.__init__(self, ME, handler, ssl_ctx)
+	self.impl = impl
+	self.soap_methods = impl.soap_services
+	self.xmlrpc_methods = impl.xmlrpc_services
+	self.get_handler = impl.get_handler
+	self.log = logging.getLogger("fedd")
+	if debug: self.handle_error = self.handle_error_debug
+	else: self.handle_error = self.handle_error_standard
+
+    def handle_error_debug(self, request=None, client_address=None):
+	print '-'*40
+	traceback.print_exc()
+	print '-'*40
+
+
+    def handle_error_standard(self, request=None, address=None):
+	"""
+	The default SSLServer prints a stack trace here.  This is a little
+	friendlier.
+	"""
+	if request or address:
+	    self.log.warn("[fedd] Error on incoming connection: %s %s" % \
+		    (request or "", address or ""))
+	else:
+	    self.log.warn("[fedd] Error on incoming connection " + \
+		    "(Likely SSL error)")
+
+class soap_handler(BaseHTTPRequestHandler):
+    """
+    Standard connection between SOAP and the fedd services in impl.
+
+    Much of this is boilerplate from 
+    http://www.xml.com/pub/a/ws/2004/01/20/salz.html
+    """
+    server_version = "ZSI/2.0 fedd/0.1 " + BaseHTTPRequestHandler.server_version
+
+    def send_xml(self, text, code=200):
+	"""Send an XML document as reply"""
+	self.send_response(code)
+	self.send_header('Content-type', 'text/xml; charset="utf-8"')
+	self.send_header('Content-Length', str(len(text)))
+	self.end_headers()
+	self.wfile.write(text)
+	self.wfile.flush()
+	self.request.socket.close()
+
+    def send_fault(self, f, code=500):
+	"""Send a SOAP encoded fault as reply"""
+	self.send_xml(f.AsSOAP(processContents="lax"), code)
+
+    def check_headers(self, ps):
+	"""Send a fault for any required envelope headers"""
+	for (uri, localname) in ps.WhatMustIUnderstand():
+	    self.send_fault(FaultFromNotUnderstood(uri, lname, 'fedd'))
+	    return False
+	return  True
+
+    def check_method(self, ps):
+	"""Confirm that this class implements the namespace and SOAP method"""
+	root = ps.body_root
+	if root.namespaceURI not in self.server.impl.soap_namespaces:
+	    self.send_fault(Fault(Fault.Client, 
+		'Unknown namespace "%s"' % root.namespaceURI))
+	    return False
+
+	if getattr(root, 'localName', None) == None:
+	    self.send_fault(Fault(Fault.Client, 'No method"'))
+	    return False
+	return True
+
+    def do_POST(self):
+	"""Treat an HTTP POST request as a SOAP service call"""
+	try:
+	    cl = int(self.headers['content-length'])
+	    data = self.rfile.read(cl)
+	    ps = ParsedSoap(data)
+	except ParseException, e:
+	    self.send_fault(Fault(Fault.Client, str(e)))
+	    return
+	except Exception, e:
+	    self.send_fault(FaultFromException(e, 0, sys.exc_info()[2]))
+	    return
+	if not self.check_headers(ps): return
+	if not self.check_method(ps): return
+	try:
+	    resp = self.soap_dispatch(ps.body_root.localName, ps,
+		    fedid(cert=self.request.get_peer_cert()))
+	except Fault, f:
+	    self.send_fault(f)
+	    resp = None
+	
+	if resp != None:
+	    sw = SoapWriter()
+	    sw.serialize(resp)
+	    self.send_xml(str(sw))
+
+    def do_GET(self):
+	"""
+	If a get handler is registered, use that to retrieve data.
+	"""
+	if self.server.get_handler:
+	    code = 200
+	    fid = fedid(cert=self.request.get_peer_cert())
+	    fname, type = self.server.get_handler(self.path, fid)
+	    if fname:
+		try:
+		    f = open(fname, "rb")
+		    size = os.path.getsize(fname)
+		except EnvironmentError, e:
+		    code = 404
+		    size = 0
+	    else:
+		code = 404
+		size = 0
+	else:
+	    code = 404
+	    size = 0
+
+	self.send_response(code)
+	if code == 200:
+	    if type:
+		self.send_header('Content-type', type)
+	    self.send_header('Content-Length', size)
+	    self.end_headers()
+	    bytes = f.read(4096)
+	    while len(bytes):
+		self.wfile.write(bytes)
+		bytes = f.read(4096)
+	    self.wfile.flush()
+	self.request.socket.close()
+
+
+    def log_request(self, code=0, size=0):
+	"""
+	Log request to the fedd logger
+	"""
+	self.server.log.info("Successful SOAP request code %d" % code)
+
+    def soap_dispatch(self, method, req, fid):
+	"""
+	The connection to the implementation, using the  method maps
+
+	The implementation provides a mapping from SOAP method name to the
+	method in the implementation that provides the service.
+	"""
+	if self.server.soap_methods.has_key(method):
+	    try:
+		return self.server.soap_methods[method](req, fid)
+	    except service_error, e:
+		de = ns0.faultType_Def(
+			(ns0.faultType_Def.schema,
+			    "FeddFaultBody")).pyclass()
+		de._code=e.code
+		de._errstr=e.code_string()
+		de._desc=e.desc
+		if  e.is_server_error():
+		    raise Fault(Fault.Server, e.code_string(), detail=de)
+		else:
+		    raise Fault(Fault.Client, e.code_string(), detail=de)
+	else:
+	    raise Fault(Fault.Client, "Unknown method: %s" % method)
+
+
+class xmlrpc_handler(BaseHTTPRequestHandler):
+    """
+    Standard connection between XMLRPC and the fedd services in impl.
+
+    Much of this is boilerplate from 
+    http://www.xml.com/pub/a/ws/2004/01/20/salz.html
+    """
+    server_version = "ZSI/2.0 fedd/0.1 " + BaseHTTPRequestHandler.server_version
+
+    def send_xml(self, text, code=200):
+	"""Send an XML document as reply"""
+	self.send_response(code)
+	self.send_header('Content-type', 'text/xml; charset="utf-8"')
+	self.send_header('Content-Length', str(len(text)))
+	self.end_headers()
+	self.wfile.write(text)
+	self.wfile.flush()
+	# Make sure to close the socket when we're done
+	self.request.socket.close()
+
+    def do_POST(self):
+	"""Treat an HTTP POST request as an XMLRPC service call"""
+	# NB: XMLRPC faults are not HTTP errors, so the code is always 200,
+	# unless an HTTP error occurs, which we don't handle.
+
+	resp = None
+	data = None
+	method = None
+	cl = int(self.headers['content-length'])
+	data = self.rfile.read(cl)
+
+	try:
+	    params, method = xmlrpclib.loads(data)
+	except xmlrpclib.ResponseError:
+	    data = xmlrpclib.dumps(xmlrpclib.Fault("Client", 
+		"Malformed request"), methodresponse=True)
+
+	if method != None:
+	    try:
+		resp = self.xmlrpc_dispatch(method, params,
+			    fedid(cert=self.request.get_peer_cert()))
+		data = xmlrpclib.dumps((resp,), encoding='UTF-8', 
+			methodresponse=True)
+	    except xmlrpclib.Fault, f:
+		data = xmlrpclib.dumps(f, methodresponse=True)
+		resp = None
+
+	self.send_xml(data)
+
+    def log_request(self, code=0, size=0):
+	"""
+	Log request to the fedd logger
+	"""
+	self.server.log.info("Successful XMLRPC request code %d" % code)
+
+
+    def xmlrpc_dispatch(self, method, req, fid):
+	"""
+	The connection to the implementation, using the  method maps
+
+	The implementation provides a mapping from XMLRPC method name to the
+	method in the implementation that provides the service.
+	"""
+	if self.server.xmlrpc_methods.has_key(method):
+	    try:
+		return self.server.xmlrpc_methods[method](req, fid)
+	    except service_error, e:
+		raise xmlrpclib.Fault(e.code_string(), e.desc)
+	else:
+	    raise xmlrpclib.Fault(100, "Unknown method: %s" % method)
Index: fedd/Makefile/federation/service_error.py
===================================================================
--- fedd/Makefile/federation/service_error.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/service_error.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,49 @@
+#!/usr/local/bin/python
+
+# This is used to make the service error reporting independent of the
+# transport.  The XMLRPC and SOAP dispatchers will convert it into
+# transport-specific errors
+class service_error(RuntimeError):
+    access = 1
+    protocol= 2
+    req = 3
+    server_config = 4
+    internal = 5
+    partial = 6
+    federant = 7
+    connect = 8
+    code_str = { 
+	access : "Access Denied",
+	protocol : "Protocol Error",
+	req : "Badly Formed Request",
+	server_config: "Server Configuration Error",
+	internal : "Internal Error",
+	partial: "Partial Embedding",
+	federant: "Federant Error",
+	connect: "Connection Error",
+    }
+    str_code = dict([ (v, k) for k, v in code_str.iteritems() ])
+    client_errors = ( req, partial)
+    server_errors = ( access, protocol, server_config, internal,
+	    federant, connect)
+
+    def __init__(self, code=None, desc=None, from_string=None):
+	self.code = code
+	self.desc = desc
+	if code == None:
+	    self.set_code_from_string(from_string)
+	RuntimeError.__init__(self, desc)
+
+    def code_string(self, code=None):
+	code = code or self.code
+	return service_error.code_str.get(code)
+    
+    def set_code_from_string(self, errstr):
+	self.code = service_error.str_code.get(errstr, service_error.internal)
+	return self.code
+    
+    def is_client_error(self):
+	return self.code in service_error.client_errors
+
+    def is_server_error(self):
+	return self.code in service_error.server_errors
Index: fedd/Makefile/federation/skeleton_access.py
===================================================================
--- fedd/Makefile/federation/skeleton_access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/skeleton_access.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,355 @@
+#!/usr/local/bin/python
+
+import os,sys
+import re
+import string
+import copy
+import pickle
+import logging
+import random
+
+from util import *
+from fedid import fedid, generate_fedid
+from authorizer import authorizer
+from service_error import service_error
+from remote_service import xmlrpc_handler, soap_handler, service_caller
+
+import topdl
+
+from access import access_base
+
+# Make log messages disappear if noone configures a fedd logger.  This is
+# something of an incantation, but basically it creates a logger object
+# registered to fedd.access if no other module above us has.  It's an extra
+# belt for the suspenders.
+class nullHandler(logging.Handler):
+    def emit(self, record): pass
+
+fl = logging.getLogger("fedd.access")
+fl.addHandler(nullHandler())
+
+
+# The plug-in itself.
+class access(access_base):
+    """
+    This is a demonstration plug-in for fedd.  It responds to all the
+    experiment_control requests and keeps internal state.  The allocations it
+    makes are simple integers associated with each valid request.  It makes use
+    of the general routines in access.access_base.
+
+    Detailed comments in the code and info at 
+    """
+
+    @staticmethod 
+    def parse_access_string(s):
+	"""
+	Parse a parenthesized string from the access db by removing the parens.
+	If the string isn't in parens, we just return it with whitespace
+	trimmed in either case.
+	"""
+	st = s.strip()
+	if st.startswith("(") and st.endswith(")"): return st[1:-1]
+	else: return st
+
+    def __init__(self, config=None, auth=None):
+        """
+	Initializer.  Pulls parameters out of the ConfigParser's access
+	section, and initializes simple internal state.  This version reads a
+	maximum integer to assign from the configuration file, while most other
+	configuration entries  are read by the base class.  
+
+	An access database in the cannonical format is also read as well as a
+	state database that is a hash of internal state.  Routines to
+	manipulate these are in the base class, but specializations appear
+	here.
+
+	The access database maps users to a simple string.  Use the sample at.
+        """
+
+	# Calling the base initializer, which reads canonical configuration
+	# information and initializes canonical members.
+	access_base.__init__(self, config, auth)
+	# Reading the maximum integer parameter from the configuration file
+	self.maxint = config.getint("access", "maxint") or 5
+	# The available integers
+	self.available_ints = set(range(0,self.maxint))
+	# This will contain the local data from a successful access request.
+	# Keys are the valid three-names, and values are the locally
+	# interpreted data.
+        self.access = { }
+	# Read the access values.  We read the accessDB in the derived class so
+	# that the derived class can specialize the reading of access info.  In
+	# this case, we gather the strings in the access db into self.access
+	# using the translator above.
+        if config.has_option("access", "accessdb"):
+	    try:
+		self.read_access(config.get("access", "accessdb"), 
+			self.parse_access_string)
+	    except EnvironmentError, e:
+		self.log.error("Cannot read %s: %s" % \
+			(config.get("access", "accessdb"), e))
+		raise e
+
+	# The base class initializer has read the state dictionary from the
+	# state file, if there is one.  The state variable includes information
+	# about each active allocation, keyed by the allocation identifier.
+	# This loop extracts the owners stored with each allocation and
+	# associates an access attribute with them.  Each owner is allowed to
+	# access each thing they own.  This is a specialization of the state
+	# handling.
+	self.state_lock.acquire()
+	for k in self.state.keys():
+	    # Add the owners
+	    for o in self.state[k].get('owners', []):
+		self.auth.set_attribute(o, fedid(hexstr=k))
+	    # The principal represented by the allocation itself is also
+	    # allowed to make accesses.
+	    self.auth.set_attribute(fedid(hexstr=k),fedid(hexstr=k))
+	    # Remove any allocated integers from the available ones
+	    if 'integer' in self.state[k]:
+		self.available_ints.discard(self.state[k]['integer'])
+	self.state_lock.release()
+
+	# This access controller does not specialize the process of looking up
+	# local information.  This aliases the lookup_access method to be
+	# easier to read.
+	self.lookup_access = self.lookup_access_base
+
+	# These dictionaries register the plug-in's local routines for handline
+	# these four messages with the server code above.  There's a version
+	# for SOAP and XMLRPC, depending on which interfaces the plugin
+	# supports.  There's rarely a technical reason not to support one or
+	# the other - the plugin code almost never deals with the transport -
+	# but if a plug-in writer wanted to disable XMLRPC, they could leave
+	# the self.xmlrpc_services dictionary empty.
+        self.soap_services = {\
+            'RequestAccess': soap_handler("RequestAccess", self.RequestAccess),
+            'ReleaseAccess': soap_handler("ReleaseAccess", self.ReleaseAccess),
+            'StartSegment': soap_handler("StartSegment", self.StartSegment),
+            'TerminateSegment': soap_handler("TerminateSegment", 
+		self.TerminateSegment),
+            }
+        self.xmlrpc_services =  {\
+            'RequestAccess': xmlrpc_handler('RequestAccess',
+                self.RequestAccess),
+            'ReleaseAccess': xmlrpc_handler('ReleaseAccess',
+                self.ReleaseAccess),
+            'StartSegment': xmlrpc_handler("StartSegment", self.StartSegment),
+            'TerminateSegment': xmlrpc_handler('TerminateSegment',
+                self.TerminateSegment),
+            }
+
+    def RequestAccess(self, req, fid):
+        """
+	Handle an access request.  Success here maps the requester into the
+	local access control space and establishes state about that user keyed
+	to a fedid.  We also save a copy of the certificate underlying that
+	fedid so this allocation can access configuration information and
+	shared parameters on the experiment controller.
+        """
+
+        # The dance to get into the request body
+        if req.has_key('RequestAccessRequestBody'):
+            req = req['RequestAccessRequestBody']
+        else:
+            raise service_error(service_error.req, "No request!?")
+
+	# Base class lookup routine.  If this fails, it throws a service
+	# exception denying access that triggers a fault response back to the
+	# caller.
+	found, match = self.lookup_access(req, fid)
+	self.log.info(
+		"[RequestAccess] Access granted to %s with local creds %s" % \
+		(match, found))
+	# Make a fedid for this allocation
+	allocID, alloc_cert = generate_fedid(subj="alloc", log=self.log)
+	aid = unicode(allocID)
+
+	# Store the data about this allocation: 
+	self.state_lock.acquire()
+	self.state[aid] = { }
+	self.state[aid]['user'] = found
+	self.state[aid]['owners'] = [ fid ]
+	self.write_state()
+	self.state_lock.release()
+	# Authorize the creating fedid and the principal representing the
+	# allocation to manipulate it.
+	self.auth.set_attribute(fid, allocID)
+	self.auth.set_attribute(allocID, allocID)
+
+	# Create a directory to stash the certificate in, ans stash it.
+	try:
+	    f = open("%s/%s.pem" % (self.certdir, aid), "w")
+	    print >>f, alloc_cert
+	    f.close()
+	except EnvironmentError, e:
+	    raise service_error(service_error.internal, 
+		    "Can't open %s/%s : %s" % (self.certdir, aid, e))
+	self.log.debug('[RequestAccess] Returning allocation ID: %s' % allocID)
+	return { 'allocID': { 'fedid': allocID } }
+
+    def ReleaseAccess(self, req, fid):
+	"""
+	Release the allocation granted earlier.  Access to the allocation is
+	checked and if valid, the state and cached certificate are destroyed.
+	"""
+        # The dance to get into the request body
+        if req.has_key('ReleaseAccessRequestBody'):
+            req = req['ReleaseAccessRequestBody']
+        else:
+            raise service_error(service_error.req, "No request!?")
+
+	# Pull a key out of the request.  One can request to delete an
+	# allocation by a local human readable name or by a fedid.  This finds
+	# both choices.
+	try:
+	    if 'localname' in req['allocID']:
+		auth_attr = aid = req['allocID']['localname']
+	    elif 'fedid' in req['allocID']:
+		aid = unicode(req['allocID']['fedid'])
+		auth_attr = req['allocID']['fedid']
+	    else:
+		raise service_error(service_error.req,
+			"Only localnames and fedids are understood")
+	except KeyError:
+	    raise service_error(service_error.req, "Badly formed request")
+
+	self.log.debug("[ReleaseAccess] deallocation requested for %s", aid)
+	#  Confirm access
+	if not self.auth.check_attribute(fid, auth_attr):
+	    self.log.debug("[ReleaseAccess] deallocation denied for %s", aid)
+	    raise service_error(service_error.access, "Access Denied")
+
+	# If there is an allocation in the state, delete it.  Note the locking.
+	self.state_lock.acquire()
+	if aid in self.state:
+	    self.log.debug("[ReleaseAccess] Found allocation for %s" %aid)
+	    del self.state[aid]
+	    self.write_state()
+	    self.state_lock.release()
+	    # And remove the access cert
+	    cf = "%s/%s.pem" % (self.certdir, aid)
+	    self.log.debug("[ReleaseAccess] Removing %s" % cf)
+	    os.remove(cf)
+	    return { 'allocID': req['allocID'] } 
+	else:
+	    self.state_lock.release()
+	    raise service_error(service_error.req, "No such allocation")
+
+    def StartSegment(self, req, fid):
+	"""
+	Start a segment.  In this simple skeleton, this means to parse the
+	request and assign an unassigned integer to it.  We store the integer
+	in the persistent state.
+	"""
+	try:
+	    req = req['StartSegmentRequestBody']
+	    # Get the request topology.  If not present, a KeyError is thrown.
+	    topref = req['segmentdescription']['topdldescription']
+	    # The fedid of the allocation we're attaching resources to
+	    auth_attr = req['allocID']['fedid']
+	except KeyError:
+	    raise service_error(server_error.req, "Badly formed request")
+
+	# String version of the allocation ID for keying
+	aid = "%s" % auth_attr
+	# Authorization check
+	if not self.auth.check_attribute(fid, auth_attr):
+	    raise service_error(service_error.access, "Access denied")
+	else:
+	    # See if this is a replay of an earlier succeeded StartSegment -
+	    # sometimes SSL kills 'em.  If so, replay the response rather than
+	    # redoing the allocation.
+	    self.state_lock.acquire()
+	    retval = self.state[aid].get('started', None)
+	    self.state_lock.release()
+	    if retval:
+		self.log.warning(
+			"[StartSegment] Duplicate StartSegment for %s: " \
+				% aid + \
+			"replaying response")
+		return retval
+
+	certfile = "%s/%s.pem" % (self.certdir, aid)
+
+	# Convert the topology into topdl data structures.  Again, the
+	# skeletion doesn't do anything with it, but this is how one parses a
+	# topology request.
+	if topref: topo = topdl.Topology(**topref)
+	else:
+	    raise service_error(service_error.req, 
+		    "Request missing segmentdescription'")
+
+	# The attributes of the request.  Not used by this plug-in, but that's
+	# where they are.
+	attrs = req.get('fedAttr', [])
+
+	# Gather connection information.  Used to send messages to those
+	# waiting.
+	connInfo = req.get('connection', [])
+
+	# Do the assignment,  A more complex plug-in would interface to the
+	# facility here to create and configure the allocation.
+	if len(self.available_ints) > 0:
+	    # NB: lock the data structure during allocation
+	    self.state_lock.acquire()
+	    assigned = random.choice([ i for i in self.available_ints])
+	    self.available_ints.discard(assigned)
+	    self.state[aid]['integer'] = assigned
+	    self.write_state()
+	    self.state_lock.release()
+	    self.log.debug("[StartSegment] Allocated %d to %s" \
+		    % (assigned, aid))
+	else:
+	    self.log.debug("[StartSegment] No remaining resources for %s" % aid)
+	    raise service_error(service_error.federant, "No available integers")
+
+	# Save the information
+	self.state_lock.acquire()
+	# It's possible that the StartSegment call gets retried (!). 
+	# if the 'started' key is in the allocation, we'll return it rather
+	# than redo the setup.  The integer allocation was saved when we made
+	# it.
+	self.state[aid]['started'] = { 
+		'allocID': req['allocID'],
+		'allocationLog': "Allocatation complete",
+		'segmentdescription': { 'topdldescription': topo.to_dict() }
+		}
+	retval = copy.deepcopy(self.state[aid]['started'])
+	self.write_state()
+	self.state_lock.release()
+
+	return retval
+
+    def TerminateSegment(self, req, fid):
+	"""
+	Remove the resources associated with th eallocation and stop the music.
+	In this example, this simply means removing the integer we allocated.
+	"""
+	# Gather the same access information as for Start Segment
+	try:
+	    req = req['TerminateSegmentRequestBody']
+	except KeyError:
+	    raise service_error(server_error.req, "Badly formed request")
+
+	auth_attr = req['allocID']['fedid']
+	aid = "%s" % auth_attr
+
+	self.log.debug("Terminate request for %s" %aid)
+	# Check authorization
+	if not self.auth.check_attribute(fid, auth_attr):
+	    raise service_error(service_error.access, "Access denied")
+
+	# Authorized: remove the integer from the allocation.  A more complex
+	# plug in would interface with the underlying facility to turn off the
+	# experiment here.
+	self.state_lock.acquire()
+	if aid in self.state:
+	    assigned = self.state[aid].get('integer', None)
+	    self.available_ints.add(assigned)
+	    if 'integer' in self.state[aid]:
+		del self.state[aid]['integer']
+	    self.write_state()
+	self.state_lock.release()
+   
+	return { 'allocID': req['allocID'] }
Index: fedd/Makefile/federation/synch_store.py
===================================================================
--- fedd/Makefile/federation/synch_store.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/synch_store.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,262 @@
+#!/usr/local/bin/python
+
+from threading import Lock, Condition
+
+class synch_store:
+    """
+    Serialized write-once dictionary.  Threads reading before the value for a
+    key is assigned wait for the assignment, though polling is an option.  Keys
+    may also be deleted, though deleting a value too closely after the set may
+    deadlock threads waiting for the assignment.
+    """
+
+    # Thrown if a key is assigned more than once
+    class CollisionError(Exception): pass
+
+    # Thrown if a non-existant key is deleted
+    class BadDeletionError(Exception): pass
+
+    # Thrown if a key is specified with a white space
+    class BadKeyError(Exception): pass
+
+    # Thrown when a variable has been revoked.
+    class RevokedKeyError(Exception): pass
+
+    # A value in the dict.  Each has its own condition (and lock) for threads
+    # to wait on.
+    class synch_value:
+	# The synch_value constructor. 
+	def __init__(self, value=None):
+	    self.value = value
+	    self.condition = Condition()
+
+	# Debugging representation
+	def __str__(self):
+	    return "value %s cond %s" % (self.value, self.condition)
+
+    # The synch_store constructor.  Note that the store has its own lock for
+    # serializing additions to the dict.
+    def __init__(self):
+	self.main_lock = Lock()
+	self.values = { }
+	self.revoked = set([])
+
+    # Set the value in the dict.  If threads are waiting, wake them.
+    def set_value(self, key, value):
+	if key.find(' ') != -1:
+	    raise self.BadKeyError("Whitespace is not allowed in keys")
+
+	self.main_lock.acquire()
+	if key in self.revoked:
+	    self.main_lock.release()
+	    raise synch_store.RevokedKeyError("Synch key %s revoked" % key)
+	if self.values.has_key(key):
+	    v = self.values[key]
+	    v.condition.acquire()
+	    self.main_lock.release()
+	    if v.value is None:
+		v.value = value
+		v.condition.notifyAll()
+		v.condition.release()
+	    else:
+		v.condition.release()
+		raise synch_store.CollisionError("%s already set" % key)
+	else:
+	    self.values[key] = synch_store.synch_value(value)
+	    self.main_lock.release()
+
+    # Ask for the value.  If wait is true, wait until the key is set, if it is
+    # not already.  If wait is false, poll.
+    def get_value(self, key, wait=True):
+	self.main_lock.acquire()
+	if key in self.revoked:
+	    self.main_lock.release()
+	    raise synch_store.RevokedKeyError("Synch key %s revoked" % key)
+	if not self.values.has_key(key):
+	    self.values[key] = synch_store.synch_value()
+	v = self.values[key]
+	v.condition.acquire()
+	self.main_lock.release()
+
+	while v.value is None:
+	    if wait: 
+		v.condition.wait()
+		v.condition.release()
+		self.main_lock.acquire()
+		if key in self.revoked:
+		    self.main_lock.release()
+		    raise synch_store.RevokedKeyError("Synch key %s revoked" % key)
+		else:
+		    self.main_lock.release()
+		v.condition.acquire()
+	    else:
+		break
+	rv = v.value
+	v.condition.release()
+	return rv
+
+    # Remove the key from the shared store.  If the key is not present or has
+    # never been set, fail.  Otherwise, wake any waiting threads (there
+    # shouldn't be any) and remove the value and key from the store.  Deleteing
+    # a revoked key removes its revocation.
+    def del_value(self, key):
+	self.main_lock.acquire()
+	if key in self.revoked:
+	    self.revoked.remove(key)
+	    self.main_lock.release()
+	else:
+	    if self.values.has_key(key):
+		v = self.values[key]
+		v.condition.acquire()
+		if v.value is not None:
+		    del self.values[key]
+		    self.main_lock.release()
+		    v.condition.notifyAll()
+		    v.condition.release()
+		else:
+		    self.main_lock.release()
+		    v.condition.release()
+		    raise synch_store.BadDeletionError("%s has not been set" \
+			    % key)
+	    else:
+		self.main_lock.release()
+		raise synch_store.BadDeletionError("%s does not exist yet" \
+			% key)
+
+    # Invlaidate this synchronization point.  Those waiting for it will get
+    # exceptions, and those trying to wait for it in the future will as well.
+    def revoke_key(self, key):
+	self.main_lock.acquire()
+	self.revoked.add(key)
+	if self.values.has_key(key):
+	    v = self.values[key]
+	    del self.values[key]
+	    self.main_lock.release()
+	    v.condition.acquire()
+	    v.condition.notifyAll()
+	    v.condition.release()
+	else:
+	    self.main_lock.release()
+
+
+    def all_keys(self):
+	self.main_lock.acquire()
+	rv = [ k for k in self.values.keys()]
+	self.main_lock.release()
+	return rv
+
+    # Save the store contents to a file in a simple way
+    def save(self, filename):
+	self.main_lock.acquire()
+	f = open(filename, 'w')
+	for k, v in self.values.items():
+	    self.values[k].condition.acquire()
+	    if self.values[k].value is not None:
+		print >> f, "%s %s" % ( k, str(self.values[k].value))
+	    else:
+		print >> f, "%s" % k
+	    self.values[k].condition.release()
+	self.main_lock.release()
+	f.close()
+
+    # Load the store contents from a file
+    def load(self, filename):
+	self.main_lock.acquire()
+	self.values = { }
+	f = open(filename, 'r')
+	for l in f:
+	    i = l.find(' ')
+	    if i != -1:	self.values[l[:i]] = synch_store.synch_value(l[i+1:])
+	    else: self.values[l] = synch_store.synch_value()
+	f.close()
+	self.main_lock.release()
+
+    # A debugging representation
+    def __str__(self):
+	rv = ""
+	self.main_lock.acquire()
+	for k, v in self.values.items():
+	    self.values[k].condition.acquire()
+	    rv += "%s: %s\n" % ( k, str(self.values[k]))
+	    self.values[k].condition.release()
+	self.main_lock.release()
+	return rv
+
+#debugging scaffolding, left here in case we have problems
+if __name__ == '__main__':
+    from threading import currentThread, Thread
+    from time import sleep
+
+    def get_val(k, ss, wait=True):
+	print "%s: getting %s" % (currentThread(), k)
+	try:
+	    v = ss.get_value(k, wait)
+	except synch_store.RevokedKeyError, e:
+	    print "Error: %s" %e
+	    return
+	print "%s: %s is %s" % (currentThread(), k, v)
+
+
+    def set_val(k, v, ss):
+	print "setting %s to %s" % (k, v)
+	try:
+	    ss.set_value(k, v)
+	except synch_store.CollisionError, e:
+	    print "Error: %s" %e
+	    return
+	except synch_store.RevokedKeyError, e:
+	    print "Error: %s" %e
+	    return
+	print "Done with set"
+
+    def del_val(k, ss):
+	print "deleteing %s" % k
+	try:
+	    ss.del_value(k)
+	except synch_store.BadDeletionError, e:
+	    print "Error: %s" %e
+	    return
+	except synch_store.RevokedKeyError, e:
+	    print "Error: %s" %e
+	    return
+	print "Done with del"
+
+    def rev_val(k, ss):
+	print "revoking %s" % k
+	try:
+	    ss.revoke_key(k)
+	except synch_store.BadDeletionError, e:
+	    print "Error: %s" %e
+	    return
+	except synch_store.RevokedKeyError, e:
+	    print "Error: %s" %e
+	    return
+	print "Done with revoking"
+
+
+    ss = synch_store()
+    ss.set_value('b', 'initial')
+    print ss
+
+    for i in range(0,10):
+	t = Thread(target=get_val, args=('a', ss, i % 2 == 1))
+	t.start()
+    t = Thread(target=set_val, args=('a', '1', ss))
+    t.start()
+    for i in range(0,10):
+	t = Thread(target=get_val, args=('a', ss))
+	t.start()
+
+    print ss
+    t = Thread(target=del_val, args=('a', ss))
+    t.start()
+    print ss
+
+    for i in range(0,10):
+	t = Thread(target=get_val, args=('c', ss))
+	t.start()
+
+    
+    t = Thread(target=rev_val, args=('c', ss))
+    t.start()
+
Index: fedd/Makefile/federation/topdl.py
===================================================================
--- fedd/Makefile/federation/topdl.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/topdl.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,860 @@
+#!/usr/local/bin/python
+
+import re
+import xml.parsers.expat
+
+from fedid import fedid as fedid_class
+
+class base:
+    @staticmethod
+    def init_class(c, arg):
+	if isinstance(arg, dict):
+	    try:
+		return c(**arg)
+	    except:
+		print "%s" % arg
+		raise
+	elif isinstance(arg, c):
+	    return arg
+	else:
+	    return None
+
+    @staticmethod
+    def make_list(a):
+	if isinstance(a, basestring) or isinstance(a, dict): return [ a ]
+	elif getattr(a, '__iter__', None): return a
+	else: return [ a ]
+
+    @staticmethod
+    def init_string(s):
+	"""
+	Force a string coercion for everything but a None.
+	"""
+	if s is not None: return "%s" % s
+	else: return None
+
+    def remove_attribute(self, key):
+	to_del = None
+	attrs = getattr(self, 'attribute', [])
+	for i, a in enumerate(attrs):
+	    if a.attribute == key:
+		to_del = i
+		break
+	
+	if to_del: del attrs[i]
+
+    def get_attribute(self, key):
+	rv = None
+	attrs = getattr(self, 'attribute', None)
+	if attrs:
+	    for a in attrs:
+		if a.attribute == key:
+		    rv = a.value
+		    break
+	return rv
+
+    def set_attribute(self, key, value):
+	attrs = getattr(self, 'attribute', None)
+	if attrs is None:
+	    return
+	for a in attrs:
+	    if a.attribute == key: 
+		a.value = value
+		break
+	else:
+	    attrs.append(Attribute(key, value))
+
+class ConsistencyError(RuntimeError): pass
+class NamespaceError(RuntimeError): pass
+
+class Attribute(base):
+    def __init__(self, attribute, value):
+	self.attribute = self.init_string(attribute)
+	self.value = self.init_string(value)
+
+    def clone(self):
+	return Attribute(attribute=self.attribute, value=self.value)
+
+    def to_dict(self):
+	return { 'attribute': self.attribute, 'value': self.value }
+
+class Capacity(base):
+    def __init__(self, rate, kind):
+	self.rate = float(rate)
+	self.kind = self.init_string(kind)
+
+    def clone(self):
+	return Capacity(rate=self.rate, kind=self.kind)
+
+    def to_dict(self):
+	return { 'rate': float(self.rate), 'kind': self.kind }
+
+class Latency(base):
+    def __init__(self, time, kind):
+	self.time = float(time)
+	self.kind = self.init_string(kind)
+
+    def clone(self):
+	return Latency(time=self.time, kind=self.kind)
+
+    def to_dict(self):
+	return { 'time': float(self.time), 'kind': self.kind }
+
+class Substrate(base):
+    def __init__(self, name, capacity=None, latency=None, attribute=[]):
+	self.name = self.init_string(name)
+	self.capacity = self.init_class(Capacity, capacity)
+	self.latency = self.init_class(Latency, latency)
+	self.attribute = [ self.init_class(Attribute, a) \
+		for a in self.make_list(attribute) ]
+	self.interfaces = [ ]
+
+    def clone(self):
+	if self.capacity: c = self.capacity.clone()
+	else: c = None
+
+	if self.latency: l = self.latency.clone()
+	else: l = None
+
+	return Substrate(name=self.name,
+		capacity=c,
+		latency=l,
+		attribute = [a.clone() for a in self.attribute])
+
+    def to_dict(self):
+	rv = { 'name': self.name }
+	if self.capacity:
+	    rv['capacity'] = self.capacity.to_dict()
+	if self.latency:
+	    rv['latency'] = self.latency.to_dict()
+	if self.attribute:
+	    rv['attribute'] = [ a.to_dict() for a in self.attribute ]
+	return rv
+
+class CPU(base):
+    def __init__(self, type, attribute=[]):
+	self.type = self.init_string(type)
+	self.attribute = [ self.init_class(Attribute, a) for a in \
+		self.make_list(attribute) ]
+
+    def clone(self):
+	return CPU(type=self.type,
+		attribute = [a.clone() for a in self.attribute])
+
+    def to_dict(self):
+	rv = { 'type': self.type}
+	if self.attribute:
+	    rv['attribute'] = [ a.to_dict() for a in self.attribute ]
+	return rv
+
+class Storage(base):
+    def __init__(self, amount, persistence, attribute=[]):
+	self.amount = float(amount)
+	self.presistence = self.init_string(persistence)
+	self.attribute = [ self.init_class(Attribute, a) \
+		for a in self.make_list(attribute) ]
+
+    def clone(self):
+	return Storage(amount=self.amount, persistence=self.persistence, 
+		attribute = [a.clone() for a in self.attribute])
+
+    def to_dict(self):
+	rv = { 'amount': float(self.amount), 'persistence': self.persistence }
+	if self.attribute:
+	    rv['attribute'] = [ a.to_dict() for a in self.attribute ]
+	return rv
+
+class OperatingSystem(base):
+    def __init__(self, name=None, version=None, distribution=None,
+	    distributionversion=None, attribute=[]):
+	self.name = self.init_string(name)
+	self.version = self.init_string(version)
+	self.distribution = self.init_string(distribution)
+	self.distributionversion = self.init_string(distributionversion)
+	self.attribute = [ self.init_class(Attribute, a) \
+		for a in self.make_list(attribute) ]
+
+    def clone(self):
+	return OperatingSystem(name=self.name,
+		version=self.version,
+		distribution=self.distribution,
+		distributionversion=self.distributionversion,
+		attribute = [ a.clone() for a in self.attribute])
+
+    def to_dict(self):
+	rv = { }
+	if self.name: rv['name'] = self.name
+	if self.version: rv['version'] = self.version
+	if self.distribution: rv['version'] = self.distribution
+	if self.distributionversion: rv['version'] = self.distributionversion
+	if self.attribute:
+	    rv['attribute'] = [ a.to_dict() for a in self.attribute ]
+	return rv
+
+class Software(base):
+    def __init__(self, location, install=None, attribute=[]):
+	self.location = self.init_string(location)
+	self.install = self.init_string(install)
+	self.attribute = [ self.init_class(Attribute, a)\
+		for a in self.make_list(attribute) ]
+
+    def clone(self):
+	return Software(location=self.location, install=self.install, 
+		attribute=[a.clone() for a in self.attribute])
+
+    def to_dict(self):
+	rv = { 'location': self.location }
+	if self.install: rv['install'] = self.install
+	if self.attribute:
+	    rv['attribute'] = [ a.to_dict() for a in self.attribute ]
+	return rv
+
+class Interface(base):
+    def __init__(self, substrate, name=None, capacity=None, latency=None,
+	    attribute=[], element=None):
+	self.name = self.init_string(name)
+
+	self.substrate = self.make_list(substrate)
+	self.capacity = self.init_class(Capacity, capacity)
+	self.latency = self.init_class(Latency, latency)
+	self.attribute = [ self.init_class(Attribute, a) \
+		for a in self.make_list(attribute) ]
+	self.element = element 
+	self.subs = [ ]
+
+    def clone(self):
+	if self.capacity: c = self.capacity.clone()
+	else: c = None
+
+	if self.latency: l = self.latency.clone()
+	else: l = None
+
+	return Interface(substrate=[s for s in self.substrate], name=self.name,
+		capacity=c, latency=l,
+		attribute = [ a.clone() for a in self.attribute])
+
+    def to_dict(self):
+	rv = { 'substrate': self.substrate, 'name': self.name }
+	if self.capacity:
+	    rv['capacity'] = self.capacity.to_dict()
+	if self.latency:
+	    rv['latency'] = self.latency.to_dict()
+	if self.attribute:
+	    rv['attribute'] = [ a.to_dict() for a in self.attribute ]
+	return rv
+
+class ID(base):
+    def __init__(self, fedid=None, uuid=None, uri=None, localname=None,
+	    kerberosUsername=None):
+	self.fedid=fedid_class(hexstr="%s" % fedid)
+	self.uuid = self.init_string(uuid)
+	self.uri = self.init_string(uri)
+	self.localname =self.init_string( localname)
+	self.kerberosUsername = self.init_string(kerberosUsername)
+
+    def clone(self):
+	return ID(self.fedid, self.uuid, self.uri, self.localname,
+		self.kerberosUsername)
+
+    def to_dict(self):
+	rv = { }
+	if self.fedid: rv['fedid'] = self.fedid
+	if self.uuid: rv['uuid'] = self.uuid
+	if self.uri: rv['uri'] = self.uri
+	if self.localname: rv['localname'] = self.localname
+	if self.kerberosUsername: rv['kerberosUsername'] = self.kerberosUsername
+	return rv
+
+class Computer(base):
+    def __init__(self, name, cpu=[], os=[], software=[], storage=[],
+	    interface=[], attribute=[]):
+	def assign_element(i):
+	    i.element = self
+
+	self.name = self.init_string(name)
+	self.cpu = [ self.init_class(CPU, c)  for c in self.make_list(cpu) ]
+	self.os = [ self.init_class(OperatingSystem, c) \
+		for c in self.make_list(os) ]
+	self.software = [ self.init_class(Software, c) \
+		for c in self.make_list(software) ]
+	self.storage = [ self.init_class(Storage, c) \
+		for c in self.make_list(storage) ]
+	self.interface = [ self.init_class(Interface, c) \
+		for c in self.make_list(interface) ]
+	self.attribute = [ self.init_class(Attribute, a) \
+		for a in self.make_list(attribute) ]
+	map(assign_element, self.interface)
+
+    def clone(self):
+	# Copy the list of names
+	return Computer(name=self.name,
+		cpu=[x.clone() for x in self.cpu],
+		os=[x.clone() for x in self.os],
+		software=[x.clone() for x in self.software],
+		storage=[x.clone() for x in self.storage],
+		interface=[x.clone() for x in self.interface],
+		attribute=[x.clone() for x in self.attribute])
+
+    def to_dict(self):
+	rv = { }
+	if self.name:
+	    rv['name'] = self.name
+	if self.cpu:
+	    rv['cpu'] = [ c.to_dict() for  c in self.cpu ]
+	if self.os:
+	    rv['os'] = [ o.to_dict() for o in self.os ]
+	if self.software:
+	    rv['software'] = [ s.to_dict() for s in self.software ]
+	if self.storage:
+	    rv['storage'] = [ s.to_dict for s in self.storage ]
+	if self.interface:
+	    rv['interface'] = [ i.to_dict() for i in self.interface ]
+	if self.attribute:
+	    rv['attribute'] = [ i.to_dict() for i in self.attribute ]
+	return { 'computer': rv }
+
+
+class Testbed(base):
+    def __init__(self, uri, type, interface=[], attribute=[]):
+	self.uri = self.init_string(uri)
+	self.type = self.init_string(type)
+	self.interface = [ self.init_class(Interface, c) \
+		for c in self.make_list(interface) ]
+	self.attribute = [ self.init_class(Attribute, c) \
+		for c in self.make_list(attribute) ]
+
+    def clone(self):
+	return Testbed(self.uri, self.type,
+		interface=[i.clone() for i in self.interface],
+		attribute=[a.cone() for a in self.attribute])
+
+    def to_dict(self):
+	rv = { }
+	if self.uri: rv['uri'] = self.uri
+	if self.type: rv['type'] = self.type
+	if self.interface:
+	    rv['interface'] = [ i.to_dict() for i in self.interface]
+	if self.attribute:
+	    rv['attribute'] = [ a.to_dict() for a in self.attribute]
+	return { 'testbed': rv }
+
+class Segment(base):
+    def __init__(self, id, type, uri, interface=[], attribute=[]):
+	self.id = self.init_class(ID, id)
+	self.type = self.init_string(type)
+	self.uri = self.init_string(uri)
+	self.interface = [ self.init_class(Interface, c) \
+		for c in self.make_list(interface) ]
+	self.attribute = [ self.init_class(Attribute, c) \
+		for c in self.make_list(attribute) ]
+
+    def clone(self):
+	return Segment(self.id.clone(), self.type, self.uri, 
+		interface=[i.clone() for i in self.interface], 
+		attribute=[a.clone() for a in self.attribute])
+
+    def to_dict(self):
+	rv = { }
+	if self.id: rv['id'] = self.id.to_dict()
+	if self.type: rv['type'] = self.type
+	if self.uri: rv['uri'] = self.uri
+	if self.interface:
+	    rv['interface'] = [ i.to_dict() for i in self.interface ]
+	if self.attribute:
+	    rv['attribute'] = [ a.to_dict() for a in self.attribute ]
+	return { 'segment': rv }
+
+
+class Other(base):
+    def __init__(self, interface=[], attribute=[]):
+	self.interface = [ self.init_class(Interface, c) \
+		for c in self.make_list(interface) ]
+	self.attribute = [ self.init_class(Attribute, c) \
+		for c in self.make_list(attribute) ]
+
+    def clone(self):
+	return Other(interface=[i.clone() for i in self.interface], 
+		attribute=[a.clone() for a in attribute])
+
+    def to_dict(self):
+	rv = {}
+	if self.interface:
+	    rv['interface'] = [ i.to_dict() for i in self.interface ]
+	if self.attribute:
+	    rv['attribute'] = [ a.to_dict() for a in self.attribute ]
+	return {'other': rv }
+
+
+class Topology(base):
+    @staticmethod
+    def init_element(e):
+	"""
+	e should be of the form { typename: args } where args is a dict full of
+	the right parameters to initialize the element.  e should have only one
+	key, but we walk e's keys in an arbitrary order and instantiate the
+	first key we know how to.
+	"""
+	classmap = {
+		'computer': Computer,
+		'testbed': Testbed,
+		'segment': Segment,
+		'other': Other,
+	    }
+
+	if isinstance(e, dict):
+	    for k in e.keys():
+		cl = classmap.get(k, None)
+		if cl: return cl(**e[k])
+	else:
+	    return e
+
+    def __init__(self, substrates=[], elements=[], attribute=[]):
+	self.substrates = [ self.init_class(Substrate, s) \
+		for s in self.make_list(substrates) ]
+	self.elements = [ self.init_element(e) \
+		for e in self.make_list(elements) ]
+	self.attribute = [ self.init_class(Attribute, c) \
+		for c in self.make_list(attribute) ]
+	self.incorporate_elements()
+
+    @staticmethod
+    def name_element_interfaces(e):
+	names = set([i.name for i in e.interface if i.name])
+	inum = 0
+	for i in [ i for i in e.interface if not i.name]:
+	    while inum < 1000:
+		n = "inf%03d" % inum
+		inum += 1
+		if n not in names:
+		    i.name = n
+		    break
+	    else:
+		raise NamespaceError("Cannot make new interface name")
+
+
+
+    def name_interfaces(self):
+	"""
+	For any interface without a name attribute, assign a unique one within
+	its element.
+	"""
+
+	for e in self.elements:
+	    self.name_element_interfaces(e)
+
+
+    def incorporate_elements(self):
+
+	# Could to this init in one gulp, but we want to look for duplicate
+	# substrate names
+	substrate_map = { }
+	for s in self.substrates:
+	    s.interfaces = [ ]
+	    if not substrate_map.has_key(s.name):
+		substrate_map[s.name] = s
+	    else:
+		raise ConsistencyError("Duplicate substrate name %s" % s.name)
+
+	for e in self.elements:
+	    self.name_element_interfaces(e)
+	    for i in e.interface:
+		i.element = e
+		i.subs = [ ]
+		for sn in i.substrate:
+		    # NB, interfaces have substrate names in their substrate
+		    # attribute.
+		    if substrate_map.has_key(sn):
+			sub = substrate_map[sn]
+			i.subs.append(sub)
+			sub.interfaces.append(i)
+		    else:
+			raise ConsistencyError("No such substrate for %s" % sn)
+
+    def clone(self):
+	return Topology(substrates=[s.clone() for s in self.substrates], 
+		elements=[e.clone() for e in self.elements],
+		attribute=[a.clone() for a in self.attribute])
+
+
+    def make_indices(self):
+	sub_index = dict([(s.name, s) for s in self.substrates])
+	elem_index = dict([(n, e) for e in self.elements for n in e.name])
+
+    def to_dict(self):
+	rv = { }
+	if self.substrates:
+	    rv['substrates'] = [ s.to_dict() for s in self.substrates ]
+	if self.elements:
+	    rv['elements'] = [ s.to_dict() for s in self.elements ]
+	if self.attribute:
+	    rv['attribute'] = [ s.to_dict() for s in self.attribute]
+	return rv
+
+def topology_from_xml(string=None, file=None, filename=None, top="topology"):
+    class parser:
+	def __init__(self, top):
+	    self.stack = [ ]
+	    self.chars = ""
+	    self.key = ""
+	    self.have_chars = False
+	    self.current = { }
+	    self.in_cdata = False
+	    self.in_top = False
+	    self.top = top
+	
+	def start_element(self, name, attrs):
+	    self.chars = ""
+	    self.have_chars = False
+	    self.key = str(name)
+
+	    if name == self.top:
+		self.in_top = True
+
+	    if self.in_top:
+		self.stack.append((self.current, self.key))
+		self.current = { }
+
+	def end_element(self, name):
+	    if self.in_top:
+		if self.have_chars:
+		    self.chars = self.chars.strip()
+		    if len(self.chars) >0:
+			addit = self.chars
+		    else:
+			addit = self.current
+		else:
+		    addit = self.current
+
+		parent, key = self.stack.pop()
+		if parent.has_key(key):
+		    if isinstance(parent[key], list):
+			parent[key].append(addit)
+		    else:
+			parent[key] = [parent[key], addit]
+		else:
+		    parent[key] = addit
+		self.current = parent
+		self.key = key
+
+	    self.chars = ""
+	    self.have_chars = False
+
+	    if name == self.top:
+		self.in_top= False
+
+	def char_data(self, data):
+	    if self.in_top:
+		self.have_chars = True
+		self.chars += data
+
+    p = parser(top=top)
+    xp = xml.parsers.expat.ParserCreate()
+
+    xp.StartElementHandler = p.start_element
+    xp.EndElementHandler = p.end_element
+    xp.CharacterDataHandler = p.char_data
+
+    num_set = len([ x for x in (string, filename, file)\
+	    if x is not None ])
+
+    if num_set != 1:
+	raise RuntimeError("Exactly one one of file, filename and string " + \
+		"must be set")
+    elif filename:
+	f = open(filename, "r")
+	xp.ParseFile(f)
+	f.close()
+    elif file:
+	xp.ParseFile(file)
+    elif string:
+	xp.Parse(string, isfinal=True)
+    else:
+	return None
+
+    return Topology(**p.current[top])
+
+def topology_to_xml(t, top=None):
+    """
+    Print the topology as XML.  This is quick and dirty, but should work for
+    many purposes.  Convert the topology to a dict and print it recursively.
+    """
+    from xml.sax.saxutils import escape
+
+    def dict_to_xml(e, top=None):
+	if top: rv = "<%s>" % top
+	else: rv = ""
+
+	for k in e.keys():
+	    if isinstance(e[k], basestring):
+		rv += "<%s>%s</%s>" % (k, escape(e[k]), k)
+	    elif isinstance(e[k], (int, float, long)):
+		rv += "<%s>%d</%s>" % (k, e[k], k)
+	    elif isinstance(e[k], dict):
+		rv += "<%s>%s</%s>" % (k, dict_to_xml(e[k]), k)
+	    elif getattr(e[k], '__iter__', None):
+		for ee in e[k]:
+		    if isinstance(ee, dict):
+			rv += "<%s>%s</%s>" % (k, dict_to_xml(ee), k)
+		    else:
+			rv += "<%s>%s</%s>" % (k, escape(ee), k)
+	    else:
+		try:
+		    rv += "<%s>%s</%s>" % (k, e[k], k)
+		except Exception:
+		    raise ConsistencyError("What is this?? %s %s" % (k, e[k]))
+	if top: rv += "</%s>" % top
+	return rv
+
+    return dict_to_xml(t.to_dict(), top)
+
+
+def topology_to_vtopo(t):
+    nodes = [ ]
+    lans = [ ]
+
+    for eidx, e in enumerate(t.elements):
+	if e.name: name = e.name
+	else: name = "unnamed_node%d" % eidx
+	
+	ips = [ ]
+	for idx, i in enumerate(e.interface):
+	    ip = i.get_attribute('ip4_address')
+	    ips.append(ip)
+	    port = "%s:%d" % (name, idx)
+	    for idx, s in enumerate(i.subs):
+		bw = 100000
+		delay = 0.0
+		if s.capacity:
+		    bw = s.capacity.rate
+		if i.capacity:
+		    bw = i.capacity.rate
+
+		if s.latency:
+		    delay = s.latency.time
+		if i.latency:
+		    bw = i.latency.time
+
+		lans.append({
+		    'member': port,
+		    'vname': s.name,
+		    'ip': ip,
+		    'vnode': name,
+		    'delay': delay,
+		    'bandwidth': bw,
+		    })
+	nodes.append({
+	    'ips': ":".join(ips),
+	    'vname': name,
+	    })
+
+    return { 'node': nodes, 'lan': lans }
+
+def to_tcl_name(n):
+    t = re.sub('-(\d+)', '(\\1)', n)
+    return t
+
+def generate_portal_command_filter(cmd, add_filter=None):
+    def rv(e):
+	s =""
+	if isinstance(e, Computer):
+	    gw = e.get_attribute('portal')
+	    if add_filter and callable(add_filter):
+		add = add_filter(e)
+	    else:
+		add = True
+	    if gw and add:
+		s = "%s ${%s}\n" % (cmd, to_tcl_name(e.name))
+	return s
+    return rv
+
+def generate_portal_image_filter(image):
+    def rv(e):
+	s =""
+	if isinstance(e, Computer):
+	    gw = e.get_attribute('portal')
+	    if gw:
+		s = "tb-set-node-os ${%s} %s\n" % (to_tcl_name(e.name), image)
+	return s
+    return rv
+
+def generate_portal_hardware_filter(type):
+    def rv(e):
+	s =""
+	if isinstance(e, Computer):
+	    gw = e.get_attribute('portal')
+	    if gw:
+		s = "tb-set-hardware ${%s} %s\n" % (to_tcl_name(e.name), type)
+	return s
+    return rv
+
+
+def topology_to_ns2(t, filters=[], routing="Manual"):
+    out = """
+set ns [new Simulator]
+source tb_compat.tcl
+
+"""
+
+    for e in t.elements:
+	rpms = ""
+	tarfiles = ""
+	if isinstance(e, Computer):
+	    name = to_tcl_name(e.name)
+	    out += "set %s [$ns node]\n" % name
+	    if e.os and len(e.os) == 1:
+		osid = e.os[0].get_attribute('osid')
+		if osid:
+		    out += "tb-set-node-os ${%s} %s\n" % (name, osid)
+	    hw = e.get_attribute('type')
+	    if hw:
+		out += "tb-set-hardware ${%s} %s\n" % (name, hw)
+	    for s in e.software:
+		if s.install:
+		    tarfiles += "%s %s " % (s.install, s.location)
+		else:
+		    rpms += "%s " % s.location
+	    if rpms:
+		out += "tb-set-node-rpms ${%s} %s\n" % (name, rpms)
+	    if tarfiles:
+		out += "tb-set-node-tarfiles ${%s} %s\n" % (name, tarfiles)
+	    startcmd = e.get_attribute('startup')
+	    if startcmd:
+		out+= 'tb-set-node-startcmd ${%s} "%s"\n' % (name, startcmd)
+	    for f in filters:
+		out += f(e)
+	    out+= "\n"
+    
+    for idx, s in enumerate(t.substrates):
+	loss = s.get_attribute('loss')
+	if s.latency: delay = s.latency.time
+	else: delay = 0
+
+	if s.capacity: rate = s.capacity.rate
+	else: rate = 100000
+	name = to_tcl_name(s.name or "sub%d" % idx)
+
+	if len(s.interfaces) > 2:
+	    # Lan
+	    members = [ to_tcl_name("${%s}") % i.element.name \
+		    for i in s.interfaces]
+	    out += 'set %s [$ns make-lan "%s" %fkb %fms ]\n' % \
+		    (name, " ".join([to_tcl_name(m) for m in members]),
+			    rate, delay)
+	    if loss:
+		"tb-set-lan-loss ${%s} %f\n" % (name, float(loss))
+
+	    for i in s.interfaces:
+		e = i.element
+		ip = i.get_attribute("ip4_address")
+		if ip:
+		    out += "tb-set-ip-lan ${%s} ${%s} %s\n" % \
+			    (to_tcl_name(e.name), name, ip)
+		if i.capacity and i.capacity.rate != rate:
+		    out += "tb-set-node-lan-bandwidth ${%s} ${%s} %fkb\n" % \
+			    (to_tcl_name(e.name), name, i.capacity.rate)
+		if i.latency and i.latency.time != delay:
+		    out += "tb-set-node-lan-delay ${%s} ${%s} %fms\n" % \
+			    (to_tcl_name(e.name), name, i.latency.time)
+		iloss = i.get_attribute('loss')
+		if loss and iloss != loss :
+		    out += "tb-set-node-lan-loss ${%s} ${%s} %f\n" % \
+			    (to_tcl_name(e.name), name, float(loss))
+	    out+= "\n"
+	elif len(s.interfaces) == 2:
+	    f = s.interfaces[0]
+	    t = s.interfaces[1]
+
+	    out += "set %s [$ns duplex-link ${%s} ${%s} %fkb %fms DropTail]\n" %\
+		    (name, to_tcl_name(f.element.name), 
+			    to_tcl_name(t.element.name), rate, delay)
+	    if loss:
+		out += "tb-set-link-loss ${%s} %f\n" % (name, float(loss))
+
+	    for i in s.interfaces:
+		lloss = i.get_attribute("loss")
+		cap_override = i.capacity and \
+			i.capacity.rate != rate
+		delay_override = i.latency and \
+			i.latency.time != delay
+		loss_override = lloss and lloss != loss
+		if cap_override or delay_override or loss_override:
+		    if i.capacity: cap = i.capacity.rate
+		    else: cap = rate
+
+		    if i.latency: delay = i.latency.time
+
+		    if lloss: loss = lloss
+		    else: loss = loss or 0.0
+
+		    out += "tb-set-link-simplex-params ${%s} ${%s} %fms %fkb %f\n"\
+			    % (name, to_tcl_name(i.element.name),
+				    delay, cap, loss)
+		ip = i.get_attribute('ip4_address')
+		if ip:
+		    out += "tb-set-ip-link ${%s} ${%s} %s\n" % \
+			    (to_tcl_name(i.element.name), name, ip)
+	    out+= "\n"
+	for f in filters:
+	    out+= f(s)
+    out+="$ns rtproto %s" % routing
+    out+="""
+$ns run
+"""
+    return out
+
+def topology_to_rspec(t, filters=[]):
+    out = '<?xml version="1.0" encoding="UTF-8"?>\n' + \
+	'<rspec xmlns="http://www.protogeni.net/resources/rspec/0.1"\n' + \
+	'\txmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n' + \
+	'\txsi:schemaLocation="http://www.protogeni.net/resources/rspec/0.1 '+ \
+	'http://www.protogeni.net/resources/rspec/0.1/request.xsd"\n' + \
+	'\ttype="request" >\n'
+
+    ifname = { }
+    ifnode = { }
+
+    for e in [e for e in t.elements if isinstance(e, Computer)]:
+	name = e.name
+	virt_type = e.get_attribute("virtualization_type") or "emulab-vnode"
+	exclusive = e.get_attribute("exclusive") or "1"
+	hw = e.get_attribute("type") or "pc";
+	slots = e.get_attribute("slots") or "1";
+	startup = e.get_attribute("startup")
+
+	extras = ""
+	if startup: extras += '\t\tstartup_command="%s"\n' % startup
+	out += '\t<node virtual_id="%s"\n\t\tvirtualization_type="%s"\n' % \
+		(name, virt_type)
+	out += '\t\texclusive="%s"' % exclusive
+	if extras: out += '\n%s' % extras
+	out += '>\n'
+	out += '\t\t<node_type type_name="%s" slots="%s"/>\n' % (hw, slots)
+	for i, ii in enumerate(e.interface):
+	    out += '\t\t<interface virtual_id="%s"/>\n' % ii.name
+	    ifnode[ii] = name
+	for f in filters:
+	    out += f(e)
+	out += '\t</node>\n'
+
+    for i, s in enumerate(t.substrates):
+	if len(s.interfaces) == 0: 
+	    continue
+	out += '\t<link virtual_id="%s" link_type="ethernet">\n' % s.name
+	if s.capacity and s.capacity.kind == "max":
+	    bwout = True
+	    out += '\t\t<bandwidth>%d</bandwidth>\n' % s.capacity.rate
+	else:
+	    bwout = False
+	if s.latency and s.latency.kind == "max":
+	    out += '\t\t<latency>%d</latency>\n' % s.latency.time
+	elif bwout:
+	    out += '\t\t<latency>0</latency>\n'
+	for ii in s.interfaces:
+	    out += ('\t\t<interface_ref virtual_node_id="%s" ' + \
+		    'virtual_interface_id="%s"/>\n') % (ifnode[ii], ii.name)
+	for f in filters:
+	    out += f(s)
+	out += '\t</link>\n'
+    out += '</rspec>\n'
+    return out
+
Index: fedd/Makefile/federation/util.py
===================================================================
--- fedd/Makefile/federation/util.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
+++ fedd/Makefile/federation/util.py	(revision 88e7f2f76ed0f9ce0f3042e8d7ae0c319b72036b)
@@ -0,0 +1,292 @@
+#!/usr/local/bin/python
+
+import re
+import string
+import logging
+import pickle
+
+import httplib
+
+from socket import sslerror
+
+from M2Crypto import SSL
+from M2Crypto.SSL import SSLError
+from fedid import fedid
+from service_error import service_error
+from urlparse import urlparse
+
+
+# If this is an old enough version of M2Crypto.SSL that has an
+# ssl_verify_callback that doesn't allow 0-length signed certs, create a
+# version of that callback that does.  This is edited from the original in
+# M2Crypto.SSL.cb.  This version also elides the printing to stderr.
+if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None):
+    from M2Crypto.SSL.Context import map
+    from M2Crypto import m2
+
+    def ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
+	unknown_issuer = [
+	    m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY,
+	    m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE,
+	    m2.X509_V_ERR_CERT_UNTRUSTED,
+	    m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
+	    ]
+	ssl_ctx = map()[ssl_ctx_ptr]
+
+	if errnum in unknown_issuer: 
+	    if ssl_ctx.get_allow_unknown_ca():
+		ok = 1
+	# CRL checking goes here...
+	if ok:
+	    if ssl_ctx.get_verify_depth() >= errdepth:
+		ok = 1
+	    else:
+		ok = 0
+	return ok
+else:
+    def ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
+	raise ValueError("This should never be called")
+
+class fedd_ssl_context(SSL.Context):
+    """
+    Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd.
+    """
+    def __init__(self, my_cert, trusted_certs=None, password=None):
+	"""
+	construct a fedd_ssl_context
+
+	@param my_cert: PEM file with my certificate in it
+	@param trusted_certs: PEM file with trusted certs in it (optional)
+	"""
+	SSL.Context.__init__(self)
+
+	# load_cert takes a callback to get a password, not a password, so if
+	# the caller provided a password, this creates a nonce callback using a
+	# lambda form.
+	if password != None and not callable(password):
+	    # This is cute.  password = lambda *args: password produces a
+	    # function object that returns itself rather than one that returns
+	    # the object itself.  This is because password is an object
+	    # reference and after the assignment it's a lambda.  So we assign
+	    # to a temp.
+	    pwd = str(password)
+	    password =lambda *args: pwd
+
+	# The calls to str below (and above) are because the underlying SSL
+	# stuff is intolerant of unicode.
+	if password != None:
+	    self.load_cert(str(my_cert), callback=password)
+	else:
+	    self.load_cert(str(my_cert))
+
+	# If no trusted certificates are specified, allow unknown CAs.
+	if trusted_certs: 
+	    self.load_verify_locations(trusted_certs)
+	    self.set_verify(SSL.verify_peer, 10)
+	else:
+	    # More legacy code.  Recent versions of M2Crypto express the
+	    # allow_unknown_ca option through a callback turned to allow it.
+	    # Older versions use a standard callback that respects the
+	    # attribute.  This should work under both regines.
+	    callb = getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', 
+		    ssl_verify_callback)
+	    self.set_allow_unknown_ca(True)
+	    self.set_verify(SSL.verify_peer, 10, callback=callb)
+
+def read_simple_accessdb(fn, auth, mask=[]):
+    """
+    Read a simple access database.  Each line is a fedid (of the form
+    fedid:hexstring) and a comma separated list of atributes to be assigned to
+    it.  This parses out the fedids and adds the attributes to the authorizer.
+    comments (preceded with a #) and blank lines are ignored.  Exceptions (e.g.
+    file exceptions and ValueErrors from badly parsed lines) are propagated.
+    """
+
+    rv = [ ]
+    lineno = 0
+    fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\
+	    "(\w+\s*(,\s*\w+)*)")
+
+    # If a single string came in, make it a list
+    if isinstance(mask, basestring): mask = [ mask ]
+
+    f = open(fn, 'r')
+    for line in f:
+	lineno += 1
+	line = line.strip()
+	if line.startswith('#') or len(line) == 0: 
+	    continue
+	m = fedid_line.match(line)
+	if m :
+	    fid = fedid(hexstr=m.group(1))
+	    for a in [ a.strip() for a in m.group(2).split(",") \
+		    if not mask or a.strip() in mask ]:
+		auth.set_attribute(fid, a.strip())
+	else:
+	    raise ValueError("Badly formatted line in accessdb: %s line %d" %\
+		    (fn, lineno))
+    f.close()
+    return rv
+	
+
+def pack_id(id):
+    """
+    Return a dictionary with the field name set by the id type.  Handy for
+    creating dictionaries to be converted to messages.
+    """
+    if isinstance(id, fedid): return { 'fedid': id }
+    elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id }
+    else: return { 'localname': id}
+
+def unpack_id(id):
+    """return id as a type determined by the key"""
+    for k in ("localname", "fedid", "uri", "kerberosUsername"):
+	if id.has_key(k): return id[k]
+    return None
+
+def set_log_level(config, sect, log):
+    """ Set the logging level to the value passed in sect of config."""
+    # The getattr sleight of hand finds the logging level constant
+    # corrersponding to the string.  We're a little paranoid to avoid user
+    # mayhem.
+    if config.has_option(sect, "log_level"):
+	level_str = config.get(sect, "log_level")
+	try:
+	    level = int(getattr(logging, level_str.upper(), -1))
+
+	    if  logging.DEBUG <= level <= logging.CRITICAL:
+		log.setLevel(level)
+	    else:
+		log.error("Bad experiment_log value: %s" % level_str)
+
+	except ValueError:
+	    log.error("Bad experiment_log value: %s" % level_str)
+
+def copy_file(src, dest, size=1024):
+    """
+    Exceedingly simple file copy.  Throws an EnvironmentError if there's a problem.
+    """
+    s = open(src,'r')
+    d = open(dest, 'w')
+
+    buf = s.read(size)
+    while buf != "":
+	d.write(buf)
+	buf = s.read(size)
+    s.close()
+    d.close()
+
+def get_url(url, cf, destdir, fn=None, max_retries=5):
+    """
+    Get data from a federated data store.  This presents the client cert/fedid
+    to the http server.  We retry up to max_retries times.
+    """
+    po = urlparse(url)
+    if not fn:
+	fn = po.path.rpartition('/')[2]
+    retries = 0
+    ok = False
+    failed_exception = None
+    while not ok and retries < 5:
+	try:
+	    conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
+		    cert_file=cf, key_file=cf)
+	    conn.putrequest('GET', po.path)
+	    conn.endheaders()
+	    response = conn.getresponse()
+
+	    lf = open("%s/%s" % (destdir, fn), "w")
+	    buf = response.read(4096)
+	    while buf:
+		lf.write(buf)
+		buf = response.read(4096)
+	    lf.close()
+	    ok = True
+	except EnvironmentError, e:
+	    failed_excpetion = e
+	    retries += 1
+	except httplib.HTTPException, e:
+	    failed_exception = e
+	    retries += 1
+	except sslerror, e:
+	    failed_exception = e
+	    retries += 1
+	except SSLError, e:
+	    failed_exception = e
+	    retries += 1
+
+    if retries > 5 and failed_exception:
+	raise failed_excpetion
+
+# Functions to manipulate composite testbed names
+def testbed_base(tb):
+    """
+    Simple code to get the base testebd name.
+    """
+    i = tb.find('/')
+    if i == -1: return tb
+    else: return tb[0:i]
+
+def testbed_suffix(tb):
+    """
+    Simple code to get a testbed suffix, if nay.  No suffix returns None.
+    """
+    i = tb.find('/')
+    if i != -1: return tb[i+1:]
+    else: return None
+
+def split_testbed(tb):
+    """
+    Return a testbed and a suffix as a tuple.  No suffix returns None for that
+    field
+    """
+
+    i = tb.find('/')
+    if i != -1: return (tb[0:i], tb[i+1:])
+    else: return (tb, None)
+
+def join_testbed(base, suffix=None):
+    """
+    Build a testbed with suffix.  If base is itself a tuple, combine them,
+    otherwise combine the two.
+    """
+    if isinstance(base, tuple):
+	if len(base) == 2:
+	    return '/'.join(base)
+	else:
+	    raise RuntimeError("Too many tuple elements for join_testbed")
+    else:
+	if suffix:
+	    return '/'.join((base, suffix))
+	else:
+	    return base
+
+def find_pickle_problem(o, st=None):
+    """
+    Debugging routine to figure out what doesn't pickle from a dict full of
+    dicts and lists.  It tries to walk down the lists and dicts and pickle each
+    atom.  If something fails to pickle, it prints an approximation of a stack
+    trace through the data structure.
+    """
+    if st is None: st = [ ]
+
+    if isinstance(o, dict):
+	for k, i in o.items():
+	    st.append(k)
+	    find_pickle_problem(i, st)
+	    st.pop()
+    elif isinstance(o, list):
+	st.append('list')
+	for i in o:
+	    find_pickle_problem(i, st)
+	st.pop()
+    else:
+	try:
+	    pickle.dumps(o)
+	except pickle.PicklingError, e:
+	    print >>sys.stderr, "<PicklingError>"
+	    print >>sys.stderr, st
+	    print >>sys.stderr, o
+	    print >>sys.stderr, e
+	    print >>sys.stderr, "</PicklingError>"
+
