Index: fedd/fedd_ftopo.py
===================================================================
--- fedd/fedd_ftopo.py	(revision 45e880df6e8b75546f71b5e3a7a48f412ae731c4)
+++ fedd/fedd_ftopo.py	(revision 6e33086c614c0f578bcd17d7f5f6ab37d236ef90)
@@ -18,4 +18,6 @@
         self.add_option("--experiment_name", dest="exp_name",
                 type="string", help="human readable experiment name")
+	self.add_option("--fresh", dest="fresh", action="store_true",
+		default=False, help="Force controller to gather fresh status")
 
 parser = ftopo_opts()
@@ -35,5 +37,5 @@
     sys.exit("specify one of --experiment_cert and --experiment_name")
 
-req = { 'experiment': exp_id }
+req = { 'experiment': exp_id, 'fresh': opts.fresh }
 
 try:
Index: fedd/fedd_info.py
===================================================================
--- fedd/fedd_info.py	(revision 45e880df6e8b75546f71b5e3a7a48f412ae731c4)
+++ fedd/fedd_info.py	(revision 6e33086c614c0f578bcd17d7f5f6ab37d236ef90)
@@ -22,4 +22,6 @@
 		    "log", "status"),
                 help="data to extract")
+	self.add_option("--fresh", dest="fresh", action="store_true",
+		default=False, help="Force controller to gather fresh status")
         
 parser = exp_data_opts()
@@ -36,5 +38,5 @@
     sys.exit("specify one of --experiment_cert and --experiment_name")
 
-req = { 'experiment': exp_id }
+req = { 'experiment': exp_id, 'fresh': opts.fresh }
 
 # If legacy data is requested, request it from the server.
Index: fedd/federation/emulab_access.py
===================================================================
--- fedd/federation/emulab_access.py	(revision 45e880df6e8b75546f71b5e3a7a48f412ae731c4)
+++ fedd/federation/emulab_access.py	(revision 6e33086c614c0f578bcd17d7f5f6ab37d236ef90)
@@ -101,10 +101,13 @@
 	    self.start_segment = proxy_emulab_segment.start_segment
 	    self.stop_segment = proxy_emulab_segment.stop_segment
+	    self.info_segment = proxy_emulab_segment.info_segment
 	elif self.access_type == 'local_emulab':
 	    self.start_segment = local_emulab_segment.start_segment
 	    self.stop_segment = local_emulab_segment.stop_segment
+	    self.info_segment = local_emulab_segment.info_segment
 	else:
 	    self.start_segment = None
 	    self.stop_segment = None
+	    self.info_segment = None
 
         self.restricted = [ ]
@@ -181,4 +184,5 @@
             'TerminateSegment': soap_handler("TerminateSegment",
 		self.TerminateSegment),
+            'InfoSegment': soap_handler("InfoSegment", self.InfoSegment),
             }
         self.xmlrpc_services =  {\
@@ -190,4 +194,5 @@
             'TerminateSegment': xmlrpc_handler('TerminateSegment',
                 self.TerminateSegment),
+            'InfoSegment': xmlrpc_handler("InfoSegment", self.InfoSegment),
             }
 
@@ -1011,26 +1016,37 @@
 	return (ename, proj, user, pubkey_base, secretkey_base, alloc_log)
 
-    def finalize_experiment(self, starter, topo, aid, alloc_id, proof):
-	"""
-	Store key bits of experiment state in the global repository, including
-	the response that may need to be replayed, and return the response.
-	"""
-	i = 0
-	t = topo.clone()
+    def decorate_topology(self, info, t):
+	"""
+	Copy the physical mapping and status onto the topology.  Used by
+	StartSegment and InfoSegment
+	"""
+	def add_new(ann, attr):
+	    for a in ann:
+		if a not in attr: attr.append(a)
+
 	# Copy the assigned names into the return topology
 	for e in t.elements:
 	    if isinstance(e, topdl.Computer):
 		if not self.create_debug:
-		    if e.name in starter.node:
-			e.localname.append("%s%s" % \
-				(starter.node[e.name][0], self.domain))
-			e.status = starter.node[e.name][1]
+		    if e.name in info.node:
+			add_new(("%s%s" % (info.node[e.name][0], self.domain),),
+				e.localname)
+			e.status = info.node[e.name][1]
 		else:
 		    # Simple debugging assignment
-		    e.localname.append("node%d%s" % (i, self.domain))
+		    add_new(("node%d%s" % (i, self.domain),), e.localname)
 		    e.status = 'active'
-		    e.operation.extend(('testop1', 'testop2'))
+		    add_new(('testop1', 'testop2'), e.operation)
 		    i += 1
 
+
+    def finalize_experiment(self, starter, topo, aid, alloc_id, proof):
+	"""
+	Store key bits of experiment state in the global repository, including
+	the response that may need to be replayed, and return the response.
+	"""
+	i = 0
+	t = topo.clone()
+	self.decorate_topology(starter, t)
 	# Grab the log (this is some anal locking, but better safe than
 	# sorry)
@@ -1246,9 +1262,10 @@
 	info = self.info_segment(keyfile=self.ssh_privkey_file,
 		debug=self.create_debug, boss=self.boss, cert=self.xmlrpc_cert)
-	info(self, user, proj, ename, gid, nonce)
+	info(self, user, proj, ename)
+	self.decorate_topology(info, topo)
 	return { 
 		'allocID': req['allocID'], 
-		'experimentdescription' { 'topdldescription' : topo.to_dict() },
+		'segmentdescription': 
+		    { 'topdldescription' : topo.to_dict() },
 		'proof': proof.to_dict(),
 		}
-
Index: fedd/federation/emulab_segment.py
===================================================================
--- fedd/federation/emulab_segment.py	(revision 45e880df6e8b75546f71b5e3a7a48f412ae731c4)
+++ fedd/federation/emulab_segment.py	(revision 6e33086c614c0f578bcd17d7f5f6ab37d236ef90)
@@ -31,4 +31,5 @@
 	self.log =  getattr(self, 'log', None) 
 	self.debug = getattr(self, 'debug', False)
+	self.node = { }
 
     def emulab_call(self, method, params):
@@ -210,4 +211,6 @@
 	"""
 
+	ev_ok = ('ISUP', 'ALWAYSUP' )
+
 	if self.debug:
 	    if self.log:
@@ -226,8 +229,9 @@
 	    if code ==0:
 		for k, v in nodes.items():
-		    print v
 		    if v.get('erole', False) and 'pnode' in v:
 			st = v.get('status', 'up')
-			if st == 'up': st = 'active'
+			ev = v.get('eventstatus', 'ISUP')
+
+			if st == 'up' and ev in ev_ok: st = 'active'
 			else: st = 'failed'
 			self.node[k] = (v['pnode'], st)
Index: fedd/federation/experiment_control.py
===================================================================
--- fedd/federation/experiment_control.py	(revision 45e880df6e8b75546f71b5e3a7a48f412ae731c4)
+++ fedd/federation/experiment_control.py	(revision 6e33086c614c0f578bcd17d7f5f6ab37d236ef90)
@@ -90,4 +90,5 @@
     call_StartSegment = service_caller('StartSegment')
     call_TerminateSegment = service_caller('TerminateSegment')
+    call_InfoSegment = service_caller('InfoSegment')
     call_Ns2Topdl = service_caller('Ns2Topdl')
 
@@ -158,4 +159,7 @@
 		or 'legacy'
 	self.auth_dir = config.get('experiment_control', 'auth_dir')
+	# XXX: document this!
+	self.info_cache_limit = \
+		config.getint('experiment_control', 'info_cache', 600)
 	if dt: self.direct_transit = [ tb.strip() for tb in dt.split(",")]
 	else: self.direct_transit = [ ]
@@ -930,4 +934,51 @@
 		return False
 
+    class info_segment(start_segment):
+	def __init__(self, debug=False, log=None, testbed="", cert_file=None,
+		cert_pwd=None, trusted_certs=None, caller=None,
+		log_collector=None):
+	    experiment_control_local.start_segment.__init__(self, debug, 
+		    log, testbed, cert_file, cert_pwd, trusted_certs, 
+		    caller, log_collector)
+
+	def __call__(self, uri, aid):
+	    req = { 'allocID': { 'fedid' : aid } }
+
+	    try:
+		self.log.debug("Calling InfoSegment at %s " % uri)
+		r = self.caller(uri, req, self.cert_file, self.cert_pwd,
+			self.trusted_certs)
+		if r.has_key('InfoSegmentResponseBody'):
+		    self.make_map(r['InfoSegmentResponseBody'])
+		    if 'proof' in r: self.proof = r['proof']
+		    self.response = r
+		else:
+		    raise service_error(service_error.internal, 
+			    "Bad response!?: %s" %r)
+		return True
+	    except service_error, e:
+		self.log.error("Info segment failed on %s: %s" % \
+			(self.testbed, e))
+		return False
+
+    def annotate_topology(self, top, data):
+	def add_new(ann, attr):
+	    for a in ann:
+		if a not in attr: attr.append(a)
+
+	# Annotate the topology with embedding info
+	for e in top.elements:
+	    if isinstance(e, topdl.Computer):
+		for s in data:
+		    ann = s.node.get(e.name, None)
+		    if ann is not None:
+			add_new(ann[0], e.localname)
+			e.status = ann[1]
+			add_new(ann[2], e.service)
+			add_new(ann[3], e.operation)
+			break
+
+
+
     def allocate_resources(self, allocated, masters, eid, expid, 
             tbparams, top, topo, tmpdir, alloc_log=None, log_collector=None, 
@@ -1046,4 +1097,5 @@
                 self.state_lock.acquire()
 		self.state[eid].status = 'failed'
+		self.state[eid].updated()
                 if self.state_filename: self.write_state()
                 self.state_lock.release()
@@ -1065,17 +1117,5 @@
 		if s.proof:
 		    proofs[s.testbed] = s.proof
-
-	    # Annotate the topology with embedding info
-	    for e in top.elements:
-		if isinstance(e, topdl.Computer):
-		    for s in starters:
-			ann = s.node.get(e.name, None)
-			if ann is not None:
-			    e.localname.extend(ann[0])
-			    e.status = ann[1]
-			    e.service.extend(ann[2])
-			    e.operation.extend(ann[3])
-			    break
-
+	    self.annotate_topology(top, starters)
             log.info("[start_segment]: Experiment %s active" % eid)
 
@@ -1092,4 +1132,5 @@
         self.state[eid] = self.state[expid]
 	self.state[eid].top = top
+	self.state[eid].updated()
 	# Append startup proofs
 	for f in self.state[eid].get_all_allocations():
@@ -1645,4 +1686,5 @@
 	    exp = self.state[key]
 	    exp.status = "starting"
+	    exp.updated()
 	    expid = exp.fedid
 	    eid = exp.localname
@@ -1980,4 +2022,5 @@
 	proofs = [copy.deepcopy(p) for k in tbparams.keys()\
 		    for p in tbparams[k].proof]
+	exp.updated()
 	if self.state_filename: 
 	    self.write_state()
@@ -2192,4 +2235,45 @@
 	    return (None, None)
 
+    def update_info(self, key, force=False):
+	top = None
+        self.state_lock.acquire()
+        if key in self.state:
+	    if force or self.state[key].older_than(self.info_cache_limit):
+		top = self.state[key].top 
+		if top is not None: top = top.clone()
+		d1, info_params, cert, d2 = \
+			self.get_segment_info(self.state[key], need_lock=False)
+	self.state_lock.release()
+
+	if top is None: return
+
+	try:
+	    tmpdir = tempfile.mkdtemp(prefix="info-")
+	except EnvironmentError:
+	    raise service_error(service_error.internal, 
+		    "Cannot create tmp dir")
+	cert_file = self.make_temp_certfile(cert, tmpdir)
+
+	data = []
+	try:
+	    for k, (uri, aid) in info_params.items():
+		info=self.info_segment(log=self.log, testbed=uri,
+			    cert_file=cert_file, cert_pwd=None,
+			    trusted_certs=self.trusted_certs,
+			    caller=self.call_InfoSegment)
+		info(uri, aid)
+		data.append(info)
+	# Clean up the tmpdir no matter what
+	finally:
+	    if tmpdir: self.remove_dirs(tmpdir)
+
+	self.annotate_topology(top, data)
+        self.state_lock.acquire()
+        if key in self.state:
+	    self.state[key].top = top
+	    self.state[key].updated()
+	    if self.state_filename: self.write_state()
+	self.state_lock.release()
+
     
     def get_info(self, req, fid):
@@ -2205,4 +2289,5 @@
         exp = req.get('experiment', None)
 	legacy = req.get('legacy', False)
+	fresh = req.get('fresh', False)
         if exp:
             if exp.has_key('fedid'):
@@ -2219,8 +2304,10 @@
         proof = self.check_experiment_access(fid, key)
 
+	self.update_info(key, fresh)
+
         self.state_lock.acquire()
         if self.state.has_key(key):
 	    rv = self.state[key].get_info()
-	    # Copy the topo if we need to generate legacy representations
+	    # Copy the topo if we need legacy annotations
 	    if legacy:
 		top = self.state[key].top 
@@ -2229,5 +2316,5 @@
 
 	# If the legacy visualization and topology representations are
-	# requested, calculate them and add them to teh return.
+	# requested, calculate them and add them to the return.
 	if legacy and rv is not None:
 	    if top is not None:
@@ -2300,9 +2387,8 @@
 		    "Experiment has no status!?")
 
-
-    def get_termination_info(self, fed_exp):
+    def get_segment_info(self, fed_exp, need_lock=True):
 	ids = []
 	term_params = { }
-	self.state_lock.acquire()
+	if need_lock: self.state_lock.acquire()
 	ids = [ x for x in (fed_exp.localname, fed_exp.fedid) if x is not None ]
 	expcert = fed_exp.identity
@@ -2315,6 +2401,14 @@
 	    aid = fed.allocID
 	    term_params[aid] = (uri, aid)
+	if need_lock: self.state_lock.release()
+	return ids, term_params, expcert, repo
+
+
+    def get_termination_info(self, fed_exp):
+	self.state_lock.acquire()
+	ids, term_params, expcert, repo = self.get_segment_info(fed_exp, False)
 	# Change the experiment state
 	fed_exp.status = 'terminating'
+	fed_exp.updated()
 	if self.state_filename: self.write_state()
 	self.state_lock.release()
@@ -2414,4 +2508,5 @@
         if fed_exp:
 	    status = self.check_termination_status(fed_exp, force)
+	    # get_termination_info updates the experiment state
             ids, term_params, expcert, repo = self.get_termination_info(fed_exp)
 	    self.deallocate_resources(term_params, expcert, status, force, 
Index: fedd/federation/experiment_info.py
===================================================================
--- fedd/federation/experiment_info.py	(revision 45e880df6e8b75546f71b5e3a7a48f412ae731c4)
+++ fedd/federation/experiment_info.py	(revision 6e33086c614c0f578bcd17d7f5f6ab37d236ef90)
@@ -2,4 +2,6 @@
 
 import copy
+from datetime import datetime, timedelta
+from numbers import Number
 
 class allocation_info:
@@ -38,4 +40,5 @@
 	self.log = []
 	self.alloc = { }
+	self.last_update = datetime.now()
 
     def add_allocation(self, a):
@@ -47,4 +50,24 @@
     def get_all_allocations(self):
 	return self.alloc.values()
+
+    def updated(self):
+	self.last_update = datetime.now()
+
+    def older_than(self, secs=None, dt=None):
+	"""
+	If the last update of this info was more than secs seconds ago, or
+	before dt (a datetime), return True.  If both secs and dt or neither is
+	given return False.  If the last update time is completelt unknown
+	(which should never happen) return True.
+	"""
+	if self.last_update is None:
+	    return True
+	elif dt is None and isinstance(secs, Number): 
+	    return self.last_update + timedelta(seconds=secs) < datetime.now()
+	elif secs is None and isinstance(dt, datetime):
+	    return self.last_update < dt
+	else:
+	    return False
+
 
     def get_info(self):
Index: fedd/federation/local_emulab_segment.py
===================================================================
--- fedd/federation/local_emulab_segment.py	(revision 45e880df6e8b75546f71b5e3a7a48f412ae731c4)
+++ fedd/federation/local_emulab_segment.py	(revision 6e33086c614c0f578bcd17d7f5f6ab37d236ef90)
@@ -21,5 +21,4 @@
 	local_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
 	emulab_segment.__init__(self, boss=boss, cert=cert)
-	self.node = { }
 
     def set_up_experiment_filespace(self, pid, eid, tmpdir):
@@ -123,2 +122,12 @@
 	return rv
 
+class info_segment(local_segment, emulab_segment):
+    def __init__(self, log=None, keyfile=None, debug=False, boss=None, 
+	    cert=None):
+	local_segment.__init__(self, log=log, keyfile=keyfile, debug=debug)
+	emulab_segment.__init__(self, boss=boss, cert=cert)
+
+    def __call__(self, parent, user, pid, eid):
+	self.log.info("[info_segment]: Getting info from %s" % eid)
+	self.get_mapping(pid,eid)
+	return True
