Index: fedd/federation/experiment_partition.py
===================================================================
--- fedd/federation/experiment_partition.py	(revision 73e7f5c740b5090ac7be4a376cdb81c6f98221a5)
+++ fedd/federation/experiment_partition.py	(revision 73e7f5c740b5090ac7be4a376cdb81c6f98221a5)
@@ -0,0 +1,589 @@
+#!/usr/local/bin/python
+
+import os,sys
+
+import re
+import random
+import string
+import subprocess
+import tempfile
+import copy
+import pickle
+import logging
+import signal
+import time
+
+import traceback
+# For parsing visualization output and splitter output
+import xml.parsers.expat
+
+from threading import Lock, Thread, Condition
+from subprocess import call, Popen, PIPE
+
+from urlparse import urlparse
+from urllib2 import urlopen
+
+from util import *
+from fedid import fedid, generate_fedid
+from remote_service import xmlrpc_handler, soap_handler, service_caller
+from service_error import service_error
+from synch_store import synch_store
+
+import topdl
+import list_log
+from ip_allocator import ip_allocator
+from ip_addr import ip_addr
+
+
+class nullHandler(logging.Handler):
+    def emit(self, record): pass
+
+fl = logging.getLogger("fedd.experiment_control")
+fl.addHandler(nullHandler())
+
+class experiment_partition: 
+    def __init__(self, auth=None, store_url=None, tbmap=None, muxmax=2):
+        """
+        Intialize the various attributes
+        """
+
+	self.log = logging.getLogger("fedd.experiment_control." + \
+		"experiment_paritition")
+	self.auth = auth
+	self.store_url = store_url
+	self.tbmap = tbmap
+	self.muxmax = muxmax
+
+
+    def new_portal_node(self, st, dt, tbparams, master, eid, myname, desthost,
+	    portal_type, iface_desc=(), conn_type="ssh", conn_attrs=[], 
+	    expid=None):
+	"""
+	Return a new internet portal node and a dict with the connectionInfo to
+	be attached.
+	"""
+	dproject = tbparams[dt].get('project', 'project')
+	ddomain = tbparams[dt].get('domain', ".example.com")
+	mdomain = tbparams[master].get('domain', '.example.com')
+	mproject = tbparams[master].get('project', 'project')
+	muser = tbparams[master].get('user', 'root')
+	smbshare = tbparams[master].get('smbshare', 'USERS')
+
+	if st == master or dt == master:
+	    active = ("%s" % (st == master))
+	else:
+	    active = ("%s" % (st > dt))
+
+	ifaces = [ ]
+	for sub, attrs in iface_desc:
+	    inf = topdl.Interface(
+		    name="inf%03d" % len(ifaces),
+		    substrate=sub,
+		    attribute=[
+			topdl.Attribute(
+			    attribute=n,
+			    value = v)
+			for n, v in attrs
+			]
+		    )
+	    ifaces.append(inf)
+	if conn_type == "ssh":
+	    try:
+		aid = tbparams[st]['allocID']['fedid']
+	    except:
+		self.log.debug("[new_portal_node] Can't get alloc id for %s?" \
+			% st)
+		aid = None
+	    info = {
+		    "type" : conn_type, 
+		    "portal": myname,
+		    'fedAttr': [ 
+			    { 'attribute': 'masterdomain', 'value': mdomain},
+			    { 'attribute': 'masterexperiment', 'value': 
+				"%s/%s" % (mproject, eid)},
+			    { 'attribute': 'active', 'value': active},
+			    # Move to SMB service description
+			    { 'attribute': 'masteruser', 'value': muser},
+			    { 'attribute': 'smbshare', 'value': smbshare},
+			],
+		    'parameter': [
+			{
+			    'name': 'peer',
+			    'key': 'fedid:%s/%s' % (expid, myname),
+			    'store': self.store_url,
+			    'type': 'output',
+			},
+			{
+			    'name': 'ssh_port',
+			    'key': 'fedid:%s/%s-port' % (expid, myname),
+			    'store': self.store_url,
+			    'type': 'output',
+			},
+			{
+			    'name': 'peer',
+			    'key': 'fedid:%s/%s' % (expid, desthost),
+			    'store': self.store_url,
+			    'type': 'input',
+			},
+			{
+			    'name': 'ssh_port',
+			    'key': 'fedid:%s/%s-port' % (expid, desthost),
+			    'store': self.store_url,
+			    'type': 'input',
+			},
+			]
+		    }
+	    # Give this allocation the rights to access the key of the
+	    # peers
+	    if aid:
+		for h in (myname, desthost):
+		    self.auth.set_attribute(aid, 'fedid:%s/%s' % (expid, h))
+		    self.auth.set_attribute(aid, 'fedid:%s/%s-port' % \
+			    (expid, h))
+	    else:
+		self.log.error("No aid for %s in new_portal_node" % st)
+	else:
+	    info = None
+
+	return (topdl.Computer(
+		name=myname,
+		attribute=[ 
+		    topdl.Attribute(attribute=n,value=v)
+			for n, v in (\
+			    ('portal', 'true'),
+			    ('portal_type', portal_type), 
+			)
+		    ],
+		interface=ifaces,
+		), info)
+
+    def new_portal_substrate(self, st, dt, eid, tbparams, expid):
+	ddomain = tbparams[dt].get('domain', ".example.com")
+	dproject = tbparams[dt].get('project', 'project')
+	tsubstrate = \
+		topdl.Substrate(name='%s-%s' % (st, dt),
+			attribute= [
+			    topdl.Attribute(
+				attribute='portal',
+				value='true')
+			    ]
+			)
+	segment_element = topdl.Segment(
+		id= tbparams[dt]['allocID'],
+		type='emulab',
+		uri = self.tbmap.get(dt, None),
+		interface=[ 
+		    topdl.Interface(
+			substrate=tsubstrate.name),
+		    ],
+		attribute = [
+		    topdl.Attribute(attribute=n, value=v)
+			for n, v in (\
+			    ('domain', ddomain),
+			    ('experiment', "%s/%s" % \
+				    (dproject, eid)),)
+		    ],
+		)
+
+	return (tsubstrate, segment_element)
+
+    def new_dragon_topo(self, idx, sub, topo, tbs, tbparams, connInfo, expid):
+	if sub.capacity is None:
+	    raise service_error(service_error.internal,
+		    "Cannot DRAGON split substrate w/o capacity")
+	segs = [ ]
+	substr = topdl.Substrate(name="dragon%d" % idx, 
+		capacity=sub.capacity.clone(),
+		attribute=[ topdl.Attribute(attribute=n, value=v)
+		    for n, v, in (\
+			    ('vlan', 'unassigned%d' % idx),)])
+	name = "dragon%d" % idx
+	store_key = 'fedid:%s/vlan%d' % (expid, idx)
+	for tb in tbs.keys():
+	    seg = topdl.Segment(
+		    id = tbparams[tb]['allocID'],
+		    type='emulab',
+		    uri = self.tbmap.get(tb, None),
+		    interface=[ 
+			topdl.Interface(
+			    substrate=substr.name),
+			],
+		    attribute=[ topdl.Attribute(
+			attribute='dragon_endpoint', 
+			value=tbparams[tb]['dragon']),
+			]
+		    )
+	    if tbparams[tb].has_key('vlans'):
+		seg.set_attribute('vlans', tbparams[tb]['vlans'])
+	    segs.append(seg)
+
+	    # Give this allocation the rights to access the key of the
+	    # vlan_id
+	    try:
+		aid = tbparams[tb]['allocID']['fedid']
+		self.auth.set_attribute(aid, store_key)
+	    except:
+		self.log.debug("[new_dragon_topo] Can't get alloc id for %s?"\
+			% tb)
+
+	connInfo[name] = [ { 
+	    'type': 'transit',
+	    'parameter': [ { 
+		'name': 'vlan_id',
+		'key': store_key,
+		'store': self.store_url,
+		'type': 'output'
+		} ]
+	    } ]
+
+	topo[name] = \
+		topdl.Topology(substrates=[substr], elements=segs,
+			attribute=[
+			    topdl.Attribute(attribute="transit", value='true'),
+			    topdl.Attribute(attribute="dynamic", value='true'),
+			    topdl.Attribute(attribute="testbed", 
+				value='dragon'),
+			    topdl.Attribute(attribute="store_keys", 
+				value=store_key),
+			    ]
+			)
+
+    def create_dragon_substrate(self, sub, topo, tbs, tbparams, master, eid,
+	    connInfo, expid=None):
+	"""
+	Add attribiutes to the various elements indicating that they are to be
+	dragon connected and create a dragon segment in topo to be
+	instantiated.
+	"""
+
+	def get_substrate_from_topo(name, t):
+	    for s in t.substrates:
+		if s.name == name: return s
+	    else: return None
+
+
+	mdomain = tbparams[master].get('domain', '.example.com')
+	mproject = tbparams[master].get('project', 'project')
+	# dn is the number of previously created dragon nets.  This routine
+	# creates a net numbered by dn
+	dn = len([x for x in topo.keys() if x.startswith('dragon')])
+	# Count the number of interfaces on this substrate in each testbed from
+	# the global topology
+	count = { }
+	node = { }
+	for e in [ i.element for i in sub.interfaces ]:
+	    tb = e.get_attribute('testbed')
+	    count[tb] = count.get(tb, 0) + 1
+	    node[tb] = i.get_attribute('ip4_address')
+
+
+	# Set the attributes in the copies that will allow setup of dragon
+	# connections.
+	for tb in tbs.keys():
+	    s = get_substrate_from_topo(sub.name, topo[tb])
+	    if s:
+		if not connInfo.has_key(tb):
+		    connInfo[tb] = [ ]
+
+		try:
+		    aid = tbparams[tb]['allocID']['fedid']
+		except:
+		    self.log.debug("[creat_dragon_substrate] " + 
+			    "Can't get alloc id for %s?" %tb)
+		    aid = None
+
+		# This may need another look, but only a service gateway will
+		# look at the active parameter, and these are only inserted to
+		# connect to the master.
+		active = "%s" % ( tb == master)
+		info = {
+			'type': 'transit',
+			'member': [ {
+			    'element': i.element.name[0], 
+			    'interface': i.name 
+			    } for i in s.interfaces \
+				    if isinstance(i.element, topdl.Computer) ],
+			'fedAttr': [ 
+			    { 'attribute': 'masterdomain', 'value': mdomain},
+			    { 'attribute': 'masterexperiment', 'value': 
+				"%s/%s" % (mproject, eid)},
+			    { 'attribute': 'active', 'value': active},
+			    ],
+			'parameter': [ {
+			    'name': 'vlan_id',
+			    'key': 'fedid:%s/vlan%d' % (expid, dn),
+			    'store': self.store_url,
+			    'type': 'input',
+			    } ]
+			}
+		if tbs.has_key(tb):
+		    info['peer'] = tbs[tb]
+		connInfo[tb].append(info)
+
+		# Give this allocation the rights to access the key of the
+		# vlan_id
+		if aid:
+		    self.auth.set_attribute(aid, 
+			    'fedid:%s/vlan%d' % (expid, dn))
+	    else:
+		raise service_error(service_error.internal,
+			"No substrate %s in testbed %s" % (sub.name, tb))
+
+	self.new_dragon_topo(dn, sub, topo, tbs, tbparams, connInfo, expid)
+
+    def insert_internet_portals(self, sub, topo, tbs, tbparams, master, eid,
+	    segment_substrate, portals, connInfo, expid):
+	# More than one testbed is on this substrate.  Insert
+	# some portals into the subtopologies.  st == source testbed,
+	# dt == destination testbed.
+	for st in tbs.keys():
+	    if not segment_substrate.has_key(st):
+		segment_substrate[st] = { }
+	    if not portals.has_key(st): 
+		portals[st] = { }
+	    if not connInfo.has_key(st):
+		connInfo[st] = [ ]
+	    for dt in [ t for t in tbs.keys() if t != st]:
+		sproject = tbparams[st].get('project', 'project')
+		dproject = tbparams[dt].get('project', 'project')
+		mproject = tbparams[master].get('project', 'project')
+		sdomain = tbparams[st].get('domain', ".example.com")
+		ddomain = tbparams[dt].get('domain', ".example.com")
+		mdomain = tbparams[master].get('domain', '.example.com')
+		muser = tbparams[master].get('user', 'root')
+		smbshare = tbparams[master].get('smbshare', 'USERS')
+		aid = tbparams[dt]['allocID']['fedid']
+		if st == master or dt == master:
+		    active = ("%s" % (st == master))
+		else:
+		    active = ("%s" %(st > dt))
+		if not segment_substrate[st].has_key(dt):
+		    # Put a substrate and a segment for the connected
+		    # testbed in there.
+		    tsubstrate, segment_element = \
+			    self.new_portal_substrate(st, dt, eid, tbparams,
+				    expid)
+		    segment_substrate[st][dt] = tsubstrate
+		    topo[st].substrates.append(tsubstrate)
+		    topo[st].elements.append(segment_element)
+
+		new_portal = False
+		if portals[st].has_key(dt):
+		    # There's a portal set up to go to this destination.
+		    # See if there's room to multiplex this connection on
+		    # it.  If so, add an interface to the portal; if not,
+		    # set up to add a portal below.
+		    # [This little festival of braces is just a pop of the
+		    # last element in the list of portals between st and
+		    # dt.]
+		    portal = portals[st][dt][-1]
+		    mux = len([ i for i in portal.interface \
+			    if not i.get_attribute('portal')])
+		    if mux == self.muxmax:
+			new_portal = True
+			portal_type = "experiment"
+			myname = "%stunnel%d" % (dt, len(portals[st][dt]))
+			desthost = "%stunnel%d" % (st.lower(), 
+				len(portals[st][dt]))
+		    else:
+			new_i = topdl.Interface(
+				substrate=sub.name,
+				attribute=[ 
+				    topdl.Attribute(
+					attribute='ip4_address', 
+					value=tbs[dt]
+				    )
+				])
+			portal.interface.append(new_i)
+		else:
+		    # First connection to this testbed, make an empty list
+		    # and set up to add the new portal below
+		    new_portal = True
+		    portals[st][dt] = [ ]
+		    myname = "%stunnel%d" % (dt, len(portals[st][dt]))
+		    desthost = "%stunnel%d" % (st.lower(), len(portals[st][dt]))
+
+		    if dt == master or st == master: portal_type = "both"
+		    else: portal_type = "experiment"
+
+		if new_portal:
+		    infs = (
+			    (segment_substrate[st][dt].name, 
+				(('portal', 'true'),)),
+			    (sub.name, 
+				(('ip4_address', tbs[dt]),))
+			)
+		    portal, info  =  self.new_portal_node(st, dt, tbparams, 
+			    master, eid, myname, desthost, portal_type,
+			    infs, conn_type="ssh", conn_attrs=[], expid=expid)
+
+		    topo[st].elements.append(portal)
+		    portals[st][dt].append(portal)
+		    connInfo[st].append(info)
+
+    def add_control_portal(self, st, dt, master, eid, topo, tbparams, connInfo, expid):
+	# Add to the master testbed
+	tsubstrate, segment_element = \
+		self.new_portal_substrate(st, dt, eid, tbparams, expid)
+	myname = "%stunnel" % dt
+	desthost = "%stunnel" % st
+
+	portal, info = self.new_portal_node(st, dt, tbparams, master,
+		eid, myname, desthost, "control", 
+		((tsubstrate.name,(('portal','true'),)),), conn_type="ssh",
+		conn_attrs=[], expid=expid)
+
+	topo[st].substrates.append(tsubstrate)
+	topo[st].elements.append(segment_element)
+	topo[st].elements.append(portal)
+	if not connInfo.has_key(st):
+	    connInfo[st] = [ ]
+	connInfo[st].append(info)
+
+    def new_dragon_portal(self, st, dt, master, eid, myip, dip, idx, 
+	    substrate, tbparams, expid):
+	# Add to the master testbed
+	myname = "%stunnel" % dt
+	desthost = "%s" % ip_addr(dip)
+
+	portal, info = self.new_portal_node(st, dt, tbparams, master,
+		eid, myname, desthost, "control", 
+		((substrate.name,(
+		    ('portal','true'),
+		    ('ip4_address', "%s" % ip_addr(myip)),)),),
+		conn_type="transit", conn_attrs=[], expid=expid)
+
+	return portal
+
+    def add_portals(self, top, topo, eid, master, tbparams, ip_allocator, 
+	    connInfo, expid):
+	"""
+	For each substrate in the main topology, find those that
+	have nodes on more than one testbed.  Insert portal nodes
+	into the copies of those substrates on the sub topologies.
+	"""
+	segment_substrate = { }
+	portals = { }
+	for s in top.substrates:
+	    # tbs will contain an ip address on this subsrate that is in
+	    # each testbed.
+	    tbs = { }
+	    for i in s.interfaces:
+		e = i.element
+		tb = e.get_attribute('testbed')
+		if tb and not tbs.has_key(tb):
+		    for i in e.interface:
+			if s in i.subs:
+			    tbs[tb]= i.get_attribute('ip4_address')
+	    if len(tbs) < 2:
+		continue
+
+	    # DRAGON will not create multi-site vlans yet
+	    if len(tbs) == 2 and \
+		    all([tbparams[x].has_key('dragon') for x in tbs]):
+		self.create_dragon_substrate(s, topo, tbs, tbparams, 
+			master, eid, connInfo, expid)
+	    else:
+		self.insert_internet_portals(s, topo, tbs, tbparams, master,
+			eid, segment_substrate, portals, connInfo, expid)
+
+	# Make sure that all the slaves have a control portal back to the
+	# master.
+	for tb in [ t for t in tbparams.keys() if t != master ]:
+	    if len([e for e in topo[tb].elements \
+		    if isinstance(e, topdl.Computer) and \
+		    e.get_attribute('portal') and \
+		    e.get_attribute('portal_type') == 'both']) == 0:
+
+		if tbparams[master].has_key('dragon') \
+			and tbparams[tb].has_key('dragon'):
+
+		    idx = len([x for x in topo.keys() \
+			    if x.startswith('dragon')])
+		    dip, leng = ip_allocator.allocate(4)
+		    dip += 1
+		    mip = dip+1
+		    csub = topdl.Substrate(
+			    name="dragon-control-%s" % tb,
+			    capacity=topdl.Capacity(100000.0, 'max'),
+			    attribute=[
+				topdl.Attribute(
+				    attribute='portal',
+				    value='true'
+				    )
+				]
+			    )
+		    seg = topdl.Segment(
+			    id= tbparams[master]['allocID'],
+			    type='emulab',
+			    uri = self.tbmap.get(master, None),
+			    interface=[ 
+				topdl.Interface(
+				    substrate=csub.name),
+				],
+			    attribute = [
+				topdl.Attribute(attribute=n, value=v)
+				    for n, v in (\
+					('domain', 
+					    tbparams[master].get('domain',
+						".example.com")),
+					('experiment', "%s/%s" % \
+						(tbparams[master].get(
+						    'project', 
+						    'project'), 
+						    eid)),)
+				],
+			    )
+		    portal = self.new_dragon_portal(tb, master,
+			    master, eid, dip, mip, idx, csub, tbparams, expid)
+		    topo[tb].substrates.append(csub)
+		    topo[tb].elements.append(portal)
+		    topo[tb].elements.append(seg)
+
+		    mcsub = csub.clone()
+		    seg = topdl.Segment(
+			    id= tbparams[tb]['allocID'],
+			    type='emulab',
+			    uri = self.tbmap.get(tb, None),
+			    interface=[ 
+				topdl.Interface(
+				    substrate=csub.name),
+				],
+			    attribute = [
+				topdl.Attribute(attribute=n, value=v)
+				    for n, v in (\
+					('domain', 
+					    tbparams[tb].get('domain',
+						".example.com")),
+					('experiment', "%s/%s" % \
+						(tbparams[tb].get('project', 
+						    'project'), 
+						    eid)),)
+				],
+			    )
+		    portal = self.new_dragon_portal(master, tb, master,
+			    eid, mip, dip, idx, mcsub, tbparams, expid)
+		    topo[master].substrates.append(mcsub)
+		    topo[master].elements.append(portal)
+		    topo[master].elements.append(seg)
+		    for t in (master, tb):
+			topo[t].incorporate_elements()
+
+		    self.create_dragon_substrate(csub, topo, 
+			    {tb: ip_addr(mip), master: ip_addr(dip)}, 
+			    tbparams, master, eid, connInfo,
+			    expid)
+		else:
+		    self.add_control_portal(master, tb, master, eid, topo, 
+			    tbparams, connInfo, expid)
+		    self.add_control_portal(tb, master, master, eid, topo, 
+			    tbparams, connInfo, expid)
+
+	# Connect the portal nodes into the topologies and clear out
+	# substrates that are not in the topologies
+	for tb in tbparams.keys():
+	    topo[tb].incorporate_elements()
+	    topo[tb].substrates = \
+		    [s for s in topo[tb].substrates \
+			if len(s.interfaces) >0]
+
