Index: fedd/federation/experiment_control.py
===================================================================
--- fedd/federation/experiment_control.py	(revision 32e7d9360c1ed976c39502898af2af20008652c1)
+++ fedd/federation/experiment_control.py	(revision 1d913e131b5a08cb73985bff1d8c3671320cef0a)
@@ -918,4 +918,5 @@
 
 	    try:
+		self.log.debug("Calling StartSegment at %s " % uri)
 		r = self.caller(uri, req, self.cert_file, self.cert_pwd,
 			self.trusted_certs)
@@ -1281,4 +1282,97 @@
 		if self.fedkit: self.add_kit(e, self.fedkit)
 
+    def new_portal_node(self, st, dt, tbparams, master, eid, myname, desthost,
+	    portal_type, portal_subst, subst=None, addr=None):
+	sproject = tbparams[st].get('project', 'project')
+	dproject = tbparams[dt].get('project', 'project')
+	mproject = tbparams[master].get('project', 'project')
+	sdomain = tbparams[st].get('domain', ".example.com")
+	ddomain = tbparams[dt].get('domain', ".example.com")
+	mdomain = tbparams[master].get('domain', '.example.com')
+	muser = tbparams[master].get('user', 'root')
+	smbshare = tbparams[master].get('smbshare', 'USERS')
+	aid = tbparams[dt]['allocID']['fedid']
+	if st == master or dt == master:
+	    active = ("%s" % (st == master))
+	else:
+	    active = ("%s" %(st > dt))
+
+	ifaces = [
+	    topdl.Interface(
+		substrate=portal_subst,
+		attribute=[
+		    topdl.Attribute(attribute='portal',
+			value='true')
+		    ]
+		),
+	    ]
+	if subst and addr:
+	    ifaces.append(
+		    topdl.Interface(
+			substrate=subst,
+			attribute=[ 
+			    topdl.Attribute(
+				attribute='ip4_address', 
+				value=addr,
+			    )
+			]
+			))
+
+
+	return topdl.Computer(
+		name=myname,
+		attribute=[ 
+		    topdl.Attribute(attribute=n,value=v)
+			for n, v in (\
+			    ('portal', 'true'),
+			    ('domain', sdomain),
+			    ('masterdomain', mdomain),
+			    ('masterexperiment', "%s/%s" % \
+				    (mproject, eid)),
+			    ('masteruser', muser),
+			    ('smbshare', smbshare),
+			    ('experiment', "%s/%s" % \
+				    (sproject, eid)),
+			    ('peer', "%s" % desthost),
+			    ('peer_segment', "%s" % aid),
+			    ('scriptdir', 
+				"/usr/local/federation/bin"),
+			    ('active', "%s" % active),
+			    ('portal_type', portal_type), 
+			    ('startup', 'sudo -H /usr/local/federation/bin/fed-tun.pl -f /proj/%s/exp/%s/tmp/%s.%s.%s%s.gw.conf >& /tmp/bridge.log' % (sproject, eid, myname.lower(), eid.lower(), sproject.lower(), sdomain.lower())))
+		    ],
+		interface=ifaces,
+		)
+
+    def new_portal_substrate(self, st, dt, eid, tbparams):
+	ddomain = tbparams[dt].get('domain', ".example.com")
+	dproject = tbparams[dt].get('project', 'project')
+	tsubstrate = \
+		topdl.Substrate(name='%s-%s' % (st, dt),
+			attribute= [
+			    topdl.Attribute(
+				attribute='portal',
+				value='true')
+			    ]
+			)
+	segment_element = topdl.Segment(
+		id= tbparams[dt]['allocID'],
+		type='emulab',
+		uri = self.tbmap.get(dt, None),
+		interface=[ 
+		    topdl.Interface(
+			substrate=tsubstrate.name),
+		    ],
+		attribute = [
+		    topdl.Attribute(attribute=n, value=v)
+			for n, v in (\
+			    ('domain', ddomain),
+			    ('experiment', "%s/%s" % \
+				    (dproject, eid)),)
+		    ],
+		)
+
+	return (tsubstrate, segment_element)
+
     def add_portals(self, top, topo, eid, master, tbparams):
 	"""
@@ -1287,4 +1381,6 @@
 	into the copies of those substrates on the sub topologies.
 	"""
+	segment_substrate = { }
+	portals = { }
 	for s in top.substrates:
 	    # tbs will contain an ip address on this subsrate that is in
@@ -1304,10 +1400,10 @@
 	    # some portals into the subtopologies.  st == source testbed,
 	    # dt == destination testbed.
-	    segment_substrate = { }
 	    for st in tbs.keys():
-		segment_substrate[st] = { }
+		if not segment_substrate.has_key(st):
+		    segment_substrate[st] = { }
+		if not portals.has_key(st): 
+		    portals[st] = { }
 		for dt in [ t for t in tbs.keys() if t != st]:
-		    myname =  "%stunnel" % dt
-		    desthost  =  "%stunnel" % st
 		    sproject = tbparams[st].get('project', 'project')
 		    dproject = tbparams[dt].get('project', 'project')
@@ -1318,61 +1414,37 @@
 		    muser = tbparams[master].get('user', 'root')
 		    smbshare = tbparams[master].get('smbshare', 'USERS')
-		    # XXX: active and type need to be unkludged
-		    active = ("%s" % (st == master))
+		    aid = tbparams[dt]['allocID']['fedid']
+		    if st == master or dt == master:
+			active = ("%s" % (st == master))
+		    else:
+			active = ("%s" %(st > dt))
 		    if not segment_substrate[st].has_key(dt):
 			# Put a substrate and a segment for the connected
 			# testbed in there.
-			tsubstrate = \
-				topdl.Substrate(name='%s-%s' % (st, dt),
-					attribute= [
-					    topdl.Attribute(
-						attribute='portal',
-						value='true')
-					    ]
-					)
-			segment_element = topdl.Segment(
-				id= tbparams[dt]['allocID'],
-				type='emulab',
-				uri = self.tbmap.get(dt, None),
-				interface=[ 
-				    topdl.Interface(
-					substrate=tsubstrate.name),
-				    ],
-				attribute = [
-				    topdl.Attribute(attribute=n, value=v)
-					for n, v in (\
-					    ('domain', ddomain),
-					    ('experiment', "%s/%s" % \
-						    (dproject, eid)),)
-				    ],
-				)
+			tsubstrate, segment_element = \
+				self.new_portal_substrate(st, dt, eid, tbparams)
 			segment_substrate[st][dt] = tsubstrate
 			topo[st].substrates.append(tsubstrate)
 			topo[st].elements.append(segment_element)
-		    portal = topdl.Computer(
-			    name="%stunnel" % dt,
-			    attribute=[ 
-				topdl.Attribute(attribute=n,value=v)
-				    for n, v in (\
-					('portal', 'true'),
-					('domain', sdomain),
-					('masterdomain', mdomain),
-					('masterexperiment', "%s/%s" % \
-						(mproject, eid)),
-					('masteruser', muser),
-					('smbshare', smbshare),
-					('experiment', "%s/%s" % \
-						(sproject, eid)),
-					('peer', "%s" % desthost),
-					('peer_segment', "%s" % \
-						tbparams[dt]['allocID']['fedid']),
-					('scriptdir', 
-					    "/usr/local/federation/bin"),
-					('active', "%s" % active),
-					('portal_type', 'both'), 
-					('startup', 'sudo -H /usr/local/federation/bin/fed-tun.pl -f /proj/%s/exp/%s/tmp/%s.%s.%s%s.gw.conf >& /tmp/bridge.log' % (sproject, eid, myname.lower(), eid.lower(), sproject.lower(), sdomain.lower())))
-				],
-			    interface=[
-				topdl.Interface(
+
+		    new_portal = False
+		    if portals[st].has_key(dt):
+			# There's a portal set up to go to this destination.
+			# See if there's room to multiples this connection on
+			# it.  If so, add an interface to the portal; if not,
+			# set up to add a portal below.
+			# [This little festival of braces is just a pop of the
+			# last element in the list of portals between st and
+			# dt.]
+			portal = portals[st][dt][-1]
+			mux = len([ i for i in portal.interface \
+				if not i.get_attribute('portal')])
+			if mux == self.muxmax:
+			    new_portal = True
+			    portal_type = "experiment"
+			    myname = "%stunnel%d" % (dt, len(portals[st][dt]))
+			    desthost = "%stunnel%d" % (st, len(portals[st][dt]))
+			else:
+			    new_i = topdl.Interface(
 				    substrate=s.name,
 				    attribute=[ 
@@ -1381,21 +1453,74 @@
 					    value=tbs[dt]
 					)
-				    ]),
-				topdl.Interface(
-				    substrate=\
-					segment_substrate[st][dt].name,
-				    attribute=[
-					topdl.Attribute(attribute='portal',
-					    value='true')
-					]
-				    ),
-				],
-			    )
-		    if self.fedkit: self.add_kit(portal, self.fedkit)
-		    if self.gatewaykit: self.add_kit(portal, self.gatewaykit)
-
-		    topo[st].elements.append(portal)
-
-	# Connect the gateway nodes into the topologies and clear out
+				    ])
+			    portal.interface.append(new_i)
+		    else:
+			# First connection to this testbed, make an empty list
+			# and set up to add the new portal below
+			new_portal = True
+			portals[st][dt] = [ ]
+			myname = "%stunnel%d" % (dt, len(portals[st][dt]))
+			desthost = "%stunnel%d" % (st, len(portals[st][dt]))
+
+			if dt == master or st == master: portal_type = "both"
+			else: portal_type = "experiment"
+
+		    if new_portal:
+			portal =  self.new_portal_node(st, dt, tbparams, 
+				master, eid, myname, desthost, portal_type,
+				segment_substrate[st][dt].name, s.name, tbs[dt])
+			if self.fedkit:
+			    self.add_kit(portal, self.fedkit)
+			if self.gatewaykit: 
+			    self.add_kit(portal, self.gatewaykit)
+
+			topo[st].elements.append(portal)
+			portals[st][dt].append(portal)
+
+	# Make sure that all the slaves have a control portal back to the
+	# master.
+	for tb in [ t for t in tbparams.keys() if t != master ]:
+	    if len([e for e in topo[tb].elements \
+		    if isinstance(e, topdl.Computer) and \
+		    e.get_attribute('portal') and \
+		    e.get_attribute('portal_type') == 'both']) == 0:
+
+		# Add to the master testbed
+		tsubstrate, segment_element = \
+			self.new_portal_substrate(master, tb, eid, tbparams)
+		myname = "%stunnel" % tb
+		desthost = "%stunnel" % master
+
+		portal = self.new_portal_node(master, tb, tbparams, master,
+			eid, myname, desthost, "control", tsubstrate.name)
+		if self.fedkit:
+		    self.add_kit(portal, self.fedkit)
+		if self.gatewaykit: 
+		    self.add_kit(portal, self.gatewaykit)
+
+		topo[master].substrates.append(tsubstrate)
+		topo[master].elements.append(segment_element)
+		topo[master].elements.append(portal)
+
+		# And to the other testbed
+
+		tsubstrate, segment_element = \
+			self.new_portal_substrate(tb, master, eid, tbparams)
+		myname = "%stunnel" % master
+		desthost = "%stunnel" % tb
+
+		portal = self.new_portal_node(tb, master, tbparams, master,
+			eid, myname, desthost, "control", tsubstrate.name)
+		if self.fedkit:
+		    self.add_kit(portal, self.fedkit)
+		if self.gatewaykit: 
+		    self.add_kit(portal, self.gatewaykit)
+
+		topo[tb].substrates.append(tsubstrate)
+		topo[tb].elements.append(segment_element)
+		topo[tb].elements.append(portal)
+
+
+	# Connect the portal nodes into the topologies and clear out
 	# substrates that are not in the topologies
 	for tb in tbparams.keys():
