Context Navigation

source: fedd/federation/experiment_control.py @ ab847bc

axis_examplecompt_changesinfo-opsversion-3.01version-3.02

Last change on this file since ab847bc was ab847bc, checked in by Ted Faber <faber@…>, 14 years ago
New syntax for testbeds that includes a /instance rider. This allows users to request multiple segments from a single testbed.
Property mode set to `100644`
File size: 80.4 KB

Line
1	#!/usr/local/bin/python
2
3	import os,sys
4
5	import re
6	import random
7	import string
8	import subprocess
9	import tempfile
10	import copy
11	import pickle
12	import logging
13	import signal
14	import time
15
16	import traceback
17	# For parsing visualization output and splitter output
18	import xml.parsers.expat
19
20	from threading import Lock, Thread, Condition
21	from subprocess import call, Popen, PIPE
22
23	from urlparse import urlparse
24	from urllib2 import urlopen
25
26	from util import *
27	from fedid import fedid, generate_fedid
28	from remote_service import xmlrpc_handler, soap_handler, service_caller
29	from service_error import service_error
30	from synch_store import synch_store
31	from experiment_partition import experiment_partition
32
33	import topdl
34	import list_log
35	from ip_allocator import ip_allocator
36	from ip_addr import ip_addr
37
38
39	class nullHandler(logging.Handler):
40	def emit(self, record): pass
41
42	fl = logging.getLogger("fedd.experiment_control")
43	fl.addHandler(nullHandler())
44
45
46	# Right now, no support for composition.
47	class federated_service:
48	def __init__(self, name, exporter=None, importers=[], params={ }, reqs=[]):
49	self.name=name
50	self.exporter=exporter
51	self.importers=importers
52	self.params = params
53	self.reqs = reqs
54
55	class experiment_control_local:
56	"""
57	Control of experiments that this system can directly access.
58
59	Includes experiment creation, termination and information dissemination.
60	Thred safe.
61	"""
62
63	class ssh_cmd_timeout(RuntimeError): pass
64
65	class thread_pool:
66	"""
67	A class to keep track of a set of threads all invoked for the same
68	task. Manages the mutual exclusion of the states.
69	"""
70	def __init__(self, nthreads):
71	"""
72	Start a pool.
73	"""
74	self.changed = Condition()
75	self.started = 0
76	self.terminated = 0
77	self.nthreads = nthreads
78
79	def acquire(self):
80	"""
81	Get the pool's lock.
82	"""
83	self.changed.acquire()
84
85	def release(self):
86	"""
87	Release the pool's lock.
88	"""
89	self.changed.release()
90
91	def wait(self, timeout = None):
92	"""
93	Wait for a pool thread to start or stop.
94	"""
95	self.changed.wait(timeout)
96
97	def start(self):
98	"""
99	Called by a pool thread to report starting.
100	"""
101	self.changed.acquire()
102	self.started += 1
103	self.changed.notifyAll()
104	self.changed.release()
105
106	def terminate(self):
107	"""
108	Called by a pool thread to report finishing.
109	"""
110	self.changed.acquire()
111	self.terminated += 1
112	self.changed.notifyAll()
113	self.changed.release()
114
115	def clear(self):
116	"""
117	Clear all pool data.
118	"""
119	self.changed.acquire()
120	self.started = 0
121	self.terminated =0
122	self.changed.notifyAll()
123	self.changed.release()
124
125	def wait_for_slot(self):
126	"""
127	Wait until we have a free slot to start another pooled thread
128	"""
129	self.acquire()
130	while self.started - self.terminated >= self.nthreads:
131	self.wait()
132	self.release()
133
134	def wait_for_all_done(self, timeout=None):
135	"""
136	Wait until all active threads finish (and at least one has
137	started). If a timeout is given, return after waiting that long
138	for termination. If all threads are done (and one has started in
139	the since the last clear()) return True, otherwise False.
140	"""
141	if timeout:
142	deadline = time.time() + timeout
143	self.acquire()
144	while self.started == 0 or self.started > self.terminated:
145	self.wait(timeout)
146	if timeout:
147	if time.time() > deadline:
148	break
149	timeout = deadline - time.time()
150	self.release()
151	return not (self.started == 0 or self.started > self.terminated)
152
153	class pooled_thread(Thread):
154	"""
155	One of a set of threads dedicated to a specific task. Uses the
156	thread_pool class above for coordination.
157	"""
158	def __init__(self, group=None, target=None, name=None, args=(),
159	kwargs={}, pdata=None, trace_file=None):
160	Thread.__init__(self, group, target, name, args, kwargs)
161	self.rv = None # Return value of the ops in this thread
162	self.exception = None # Exception that terminated this thread
163	self.target=target # Target function to run on start()
164	self.args = args # Args to pass to target
165	self.kwargs = kwargs # Additional kw args
166	self.pdata = pdata # thread_pool for this class
167	# Logger for this thread
168	self.log = logging.getLogger("fedd.experiment_control")
169
170	def run(self):
171	"""
172	Emulate Thread.run, except add pool data manipulation and error
173	logging.
174	"""
175	if self.pdata:
176	self.pdata.start()
177
178	if self.target:
179	try:
180	self.rv = self.target(self.args, *self.kwargs)
181	except service_error, s:
182	self.exception = s
183	self.log.error("Thread exception: %s %s" % \
184	(s.code_string(), s.desc))
185	except:
186	self.exception = sys.exc_info()[1]
187	self.log.error(("Unexpected thread exception: %s" +\
188	"Trace %s") % (self.exception,\
189	traceback.format_exc()))
190	if self.pdata:
191	self.pdata.terminate()
192
193	call_RequestAccess = service_caller('RequestAccess')
194	call_ReleaseAccess = service_caller('ReleaseAccess')
195	call_StartSegment = service_caller('StartSegment')
196	call_TerminateSegment = service_caller('TerminateSegment')
197	call_Ns2Topdl = service_caller('Ns2Topdl')
198
199	def __init__(self, config=None, auth=None):
200	"""
201	Intialize the various attributes, most from the config object
202	"""
203
204	def parse_tarfile_list(tf):
205	"""
206	Parse a tarfile list from the configuration. This is a set of
207	paths and tarfiles separated by spaces.
208	"""
209	rv = [ ]
210	if tf is not None:
211	tl = tf.split()
212	while len(tl) > 1:
213	p, t = tl[0:2]
214	del tl[0:2]
215	rv.append((p, t))
216	return rv
217
218	self.thread_with_rv = experiment_control_local.pooled_thread
219	self.thread_pool = experiment_control_local.thread_pool
220	self.list_log = list_log.list_log
221
222	self.cert_file = config.get("experiment_control", "cert_file")
223	if self.cert_file:
224	self.cert_pwd = config.get("experiment_control", "cert_pwd")
225	else:
226	self.cert_file = config.get("globals", "cert_file")
227	self.cert_pwd = config.get("globals", "cert_pwd")
228
229	self.trusted_certs = config.get("experiment_control", "trusted_certs") \
230	or config.get("globals", "trusted_certs")
231
232	self.repodir = config.get("experiment_control", "repodir")
233	self.repo_url = config.get("experiment_control", "repo_url",
234	"https://users.isi.deterlab.net:23235");
235
236	self.exp_stem = "fed-stem"
237	self.log = logging.getLogger("fedd.experiment_control")
238	set_log_level(config, "experiment_control", self.log)
239	self.muxmax = 2
240	self.nthreads = 10
241	self.randomize_experiments = False
242
243	self.splitter = None
244	self.ssh_keygen = "/usr/bin/ssh-keygen"
245	self.ssh_identity_file = None
246
247
248	self.debug = config.getboolean("experiment_control", "create_debug")
249	self.cleanup = not config.getboolean("experiment_control",
250	"leave_tmpfiles")
251	self.state_filename = config.get("experiment_control",
252	"experiment_state")
253	self.store_filename = config.get("experiment_control",
254	"synch_store")
255	self.store_url = config.get("experiment_control", "store_url")
256	self.splitter_url = config.get("experiment_control", "ns2topdl_uri")
257	self.fedkit = parse_tarfile_list(\
258	config.get("experiment_control", "fedkit"))
259	self.gatewaykit = parse_tarfile_list(\
260	config.get("experiment_control", "gatewaykit"))
261	accessdb_file = config.get("experiment_control", "accessdb")
262
263	self.ssh_pubkey_file = config.get("experiment_control",
264	"ssh_pubkey_file")
265	self.ssh_privkey_file = config.get("experiment_control",
266	"ssh_privkey_file")
267	# NB for internal master/slave ops, not experiment setup
268	self.ssh_type = config.get("experiment_control", "sshkeytype", "rsa")
269
270	self.overrides = set([])
271	ovr = config.get('experiment_control', 'overrides')
272	if ovr:
273	for o in ovr.split(","):
274	o = o.strip()
275	if o.startswith('fedid:'): o = o[len('fedid:'):]
276	self.overrides.add(fedid(hexstr=o))
277
278	self.state = { }
279	self.state_lock = Lock()
280	self.tclsh = "/usr/local/bin/otclsh"
281	self.tcl_splitter = config.get("ns2topdl", "tcl_splitter") or \
282	config.get("experiment_control", "tcl_splitter",
283	"/usr/testbed/lib/ns2ir/parse.tcl")
284	mapdb_file = config.get("experiment_control", "mapdb")
285	self.trace_file = sys.stderr
286
287	self.def_expstart = \
288	"sudo -H /bin/sh /usr/local/federation/bin/federate.sh >& " +\
289	"/tmp/federate";
290	self.def_mexpstart = "sudo -H /usr/local/federation/bin/make_hosts " +\
291	"FEDDIR/hosts";
292	self.def_gwstart = \
293	"sudo -H /usr/local/federation/bin/fed-tun.pl -f GWCONF>& " +\
294	"/tmp/bridge.log";
295	self.def_mgwstart = \
296	"sudo -H /usr/local/federation/bin/fed-tun.pl -f GWCONF >& " +\
297	"/tmp/bridge.log";
298	self.def_gwimage = "FBSD61-TUNNEL2";
299	self.def_gwtype = "pc";
300	self.local_access = { }
301
302	if auth:
303	self.auth = auth
304	else:
305	self.log.error(\
306	"[access]: No authorizer initialized, creating local one.")
307	auth = authorizer()
308
309
310	if self.ssh_pubkey_file:
311	try:
312	f = open(self.ssh_pubkey_file, 'r')
313	self.ssh_pubkey = f.read()
314	f.close()
315	except IOError:
316	raise service_error(service_error.internal,
317	"Cannot read sshpubkey")
318	else:
319	raise service_error(service_error.internal,
320	"No SSH public key file?")
321
322	if not self.ssh_privkey_file:
323	raise service_error(service_error.internal,
324	"No SSH public key file?")
325
326
327	if mapdb_file:
328	self.read_mapdb(mapdb_file)
329	else:
330	self.log.warn("[experiment_control] No testbed map, using defaults")
331	self.tbmap = {
332	'deter':'https://users.isi.deterlab.net:23235',
333	'emulab':'https://users.isi.deterlab.net:23236',
334	'ucb':'https://users.isi.deterlab.net:23237',
335	}
336
337	if accessdb_file:
338	self.read_accessdb(accessdb_file)
339	else:
340	raise service_error(service_error.internal,
341	"No accessdb specified in config")
342
343	# Grab saved state. OK to do this w/o locking because it's read only
344	# and only one thread should be in existence that can see self.state at
345	# this point.
346	if self.state_filename:
347	self.read_state()
348
349	if self.store_filename:
350	self.read_store()
351	else:
352	self.log.warning("No saved synch store")
353	self.synch_store = synch_store
354
355	# Dispatch tables
356	self.soap_services = {\
357	'New': soap_handler('New', self.new_experiment),
358	'Create': soap_handler('Create', self.create_experiment),
359	'Vtopo': soap_handler('Vtopo', self.get_vtopo),
360	'Vis': soap_handler('Vis', self.get_vis),
361	'Info': soap_handler('Info', self.get_info),
362	'MultiInfo': soap_handler('MultiInfo', self.get_multi_info),
363	'Terminate': soap_handler('Terminate',
364	self.terminate_experiment),
365	'GetValue': soap_handler('GetValue', self.GetValue),
366	'SetValue': soap_handler('SetValue', self.SetValue),
367	}
368
369	self.xmlrpc_services = {\
370	'New': xmlrpc_handler('New', self.new_experiment),
371	'Create': xmlrpc_handler('Create', self.create_experiment),
372	'Vtopo': xmlrpc_handler('Vtopo', self.get_vtopo),
373	'Vis': xmlrpc_handler('Vis', self.get_vis),
374	'Info': xmlrpc_handler('Info', self.get_info),
375	'MultiInfo': xmlrpc_handler('MultiInfo', self.get_multi_info),
376	'Terminate': xmlrpc_handler('Terminate',
377	self.terminate_experiment),
378	'GetValue': xmlrpc_handler('GetValue', self.GetValue),
379	'SetValue': xmlrpc_handler('SetValue', self.SetValue),
380	}
381
382	# Call while holding self.state_lock
383	def write_state(self):
384	"""
385	Write a new copy of experiment state after copying the existing state
386	to a backup.
387
388	State format is a simple pickling of the state dictionary.
389	"""
390	if os.access(self.state_filename, os.W_OK):
391	copy_file(self.state_filename, \
392	"%s.bak" % self.state_filename)
393	try:
394	f = open(self.state_filename, 'w')
395	pickle.dump(self.state, f)
396	except IOError, e:
397	self.log.error("Can't write file %s: %s" % \
398	(self.state_filename, e))
399	except pickle.PicklingError, e:
400	self.log.error("Pickling problem: %s" % e)
401	except TypeError, e:
402	self.log.error("Pickling problem (TypeError): %s" % e)
403
404	@staticmethod
405	def get_alloc_ids(state):
406	"""
407	Pull the fedids of the identifiers of each allocation from the
408	state. Again, a dict dive that's best isolated.
409
410	Used by read_store and read state
411	"""
412
413	return [ f['allocID']['fedid']
414	for f in state.get('federant',[]) \
415	if f.has_key('allocID') and \
416	f['allocID'].has_key('fedid')]
417
418	# Call while holding self.state_lock
419	def read_state(self):
420	"""
421	Read a new copy of experiment state. Old state is overwritten.
422
423	State format is a simple pickling of the state dictionary.
424	"""
425
426	def get_experiment_id(state):
427	"""
428	Pull the fedid experimentID out of the saved state. This is kind
429	of a gross walk through the dict.
430	"""
431
432	if state.has_key('experimentID'):
433	for e in state['experimentID']:
434	if e.has_key('fedid'):
435	return e['fedid']
436	else:
437	return None
438	else:
439	return None
440
441	try:
442	f = open(self.state_filename, "r")
443	self.state = pickle.load(f)
444	self.log.debug("[read_state]: Read state from %s" % \
445	self.state_filename)
446	except IOError, e:
447	self.log.warning("[read_state]: No saved state: Can't open %s: %s"\
448	% (self.state_filename, e))
449	except pickle.UnpicklingError, e:
450	self.log.warning(("[read_state]: No saved state: " + \
451	"Unpickling failed: %s") % e)
452
453	for s in self.state.values():
454	try:
455
456	eid = get_experiment_id(s)
457	if eid :
458	# Give the owner rights to the experiment
459	self.auth.set_attribute(s['owner'], eid)
460	# And holders of the eid as well
461	self.auth.set_attribute(eid, eid)
462	# allow overrides to control experiments as well
463	for o in self.overrides:
464	self.auth.set_attribute(o, eid)
465	# Set permissions to allow reading of the software repo, if
466	# any, as well.
467	for a in self.get_alloc_ids(s):
468	self.auth.set_attribute(a, 'repo/%s' % eid)
469	else:
470	raise KeyError("No experiment id")
471	except KeyError, e:
472	self.log.warning("[read_state]: State ownership or identity " +\
473	"misformatted in %s: %s" % (self.state_filename, e))
474
475
476	def read_accessdb(self, accessdb_file):
477	"""
478	Read the mapping from fedids that can create experiments to their name
479	in the 3-level access namespace. All will be asserted from this
480	testbed and can include the local username and porject that will be
481	asserted on their behalf by this fedd. Each fedid is also added to the
482	authorization system with the "create" attribute.
483	"""
484	self.accessdb = {}
485	# These are the regexps for parsing the db
486	name_expr = "[" + string.ascii_letters + string.digits + "\.\-]+"
487	project_line = re.compile("^\s*fedid:([" + string.hexdigits + "]+)"+ \
488	"\s->\(\s("+name_expr+")\s,\s("+name_expr+")\s\)\s$")
489	user_line = re.compile("^\s*fedid:([" + string.hexdigits + "]+)"+ \
490	"\s->\s(" + name_expr + ")\s*$")
491	lineno = 0
492
493	# Parse the mappings and store in self.authdb, a dict of
494	# fedid -> (proj, user)
495	try:
496	f = open(accessdb_file, "r")
497	for line in f:
498	lineno += 1
499	line = line.strip()
500	if len(line) == 0 or line.startswith('#'):
501	continue
502	m = project_line.match(line)
503	if m:
504	fid = fedid(hexstr=m.group(1))
505	project, user = m.group(2,3)
506	if not self.accessdb.has_key(fid):
507	self.accessdb[fid] = []
508	self.accessdb[fid].append((project, user))
509	continue
510
511	m = user_line.match(line)
512	if m:
513	fid = fedid(hexstr=m.group(1))
514	project = None
515	user = m.group(2)
516	if not self.accessdb.has_key(fid):
517	self.accessdb[fid] = []
518	self.accessdb[fid].append((project, user))
519	continue
520	self.log.warn("[experiment_control] Error parsing access " +\
521	"db %s at line %d" % (accessdb_file, lineno))
522	except IOError:
523	raise service_error(service_error.internal,
524	("Error opening/reading %s as experiment " +\
525	"control accessdb") % accessdb_file)
526	f.close()
527
528	# Initialize the authorization attributes
529	for fid in self.accessdb.keys():
530	self.auth.set_attribute(fid, 'create')
531	self.auth.set_attribute(fid, 'new')
532
533	def read_mapdb(self, file):
534	"""
535	Read a simple colon separated list of mappings for the
536	label-to-testbed-URL mappings. Clears or creates self.tbmap.
537	"""
538
539	self.tbmap = { }
540	lineno =0
541	try:
542	f = open(file, "r")
543	for line in f:
544	lineno += 1
545	line = line.strip()
546	if line.startswith('#') or len(line) == 0:
547	continue
548	try:
549	label, url = line.split(':', 1)
550	self.tbmap[label] = url
551	except ValueError, e:
552	self.log.warn("[read_mapdb] Ignored bad line (%d) in " +\
553	"map db: %s %s" % (lineno, line, e))
554	except IOError, e:
555	self.log.warning("[read_mapdb]: No saved map database: Can't " +\
556	"open %s: %s" % (file, e))
557	f.close()
558
559	def read_store(self):
560	try:
561	self.synch_store = synch_store()
562	self.synch_store.load(self.store_filename)
563	self.log.debug("[read_store]: Read store from %s" % \
564	self.store_filename)
565	except IOError, e:
566	self.log.warning("[read_store]: No saved store: Can't open %s: %s"\
567	% (self.state_filename, e))
568	self.synch_store = synch_store()
569
570	# Set the initial permissions on data in the store. XXX: This ad hoc
571	# authorization attribute initialization is getting out of hand.
572	for k in self.synch_store.all_keys():
573	try:
574	if k.startswith('fedid:'):
575	fid = fedid(hexstr=k[6:46])
576	if self.state.has_key(fid):
577	for a in self.get_alloc_ids(self.state[fid]):
578	self.auth.set_attribute(a, k)
579	except ValueError, e:
580	self.log.warn("Cannot deduce permissions for %s" % k)
581
582
583	def write_store(self):
584	"""
585	Write a new copy of synch_store after writing current state
586	to a backup. We use the internal synch_store pickle method to avoid
587	incinsistent data.
588
589	State format is a simple pickling of the store.
590	"""
591	if os.access(self.store_filename, os.W_OK):
592	copy_file(self.store_filename, \
593	"%s.bak" % self.store_filename)
594	try:
595	self.synch_store.save(self.store_filename)
596	except IOError, e:
597	self.log.error("Can't write file %s: %s" % \
598	(self.store_filename, e))
599	except TypeError, e:
600	self.log.error("Pickling problem (TypeError): %s" % e)
601
602
603	def generate_ssh_keys(self, dest, type="rsa" ):
604	"""
605	Generate a set of keys for the gateways to use to talk.
606
607	Keys are of type type and are stored in the required dest file.
608	"""
609	valid_types = ("rsa", "dsa")
610	t = type.lower();
611	if t not in valid_types: raise ValueError
612	cmd = [self.ssh_keygen, '-t', t, '-N', '', '-f', dest]
613
614	try:
615	trace = open("/dev/null", "w")
616	except IOError:
617	raise service_error(service_error.internal,
618	"Cannot open /dev/null??");
619
620	# May raise CalledProcessError
621	self.log.debug("[generate_ssh_keys]: %s" % " ".join(cmd))
622	rv = call(cmd, stdout=trace, stderr=trace, close_fds=True)
623	if rv != 0:
624	raise service_error(service_error.internal,
625	"Cannot generate nonce ssh keys. %s return code %d" \
626	% (self.ssh_keygen, rv))
627
628	def gentopo(self, str):
629	"""
630	Generate the topology dtat structure from the splitter's XML
631	representation of it.
632
633	The topology XML looks like:
634	<experiment>
635	<nodes>
636	<node><vname></vname><ips>ip1:ip2</ips></node>
637	</nodes>
638	<lans>
639	<lan>
640	<vname></vname><vnode></vnode><ip></ip>
641	<bandwidth></bandwidth><member>node:port</member>
642	</lan>
643	</lans>
644	"""
645	class topo_parse:
646	"""
647	Parse the topology XML and create the dats structure.
648	"""
649	def __init__(self):
650	# Typing of the subelements for data conversion
651	self.str_subelements = ('vname', 'vnode', 'ips', 'ip', 'member')
652	self.int_subelements = ( 'bandwidth',)
653	self.float_subelements = ( 'delay',)
654	# The final data structure
655	self.nodes = [ ]
656	self.lans = [ ]
657	self.topo = { \
658	'node': self.nodes,\
659	'lan' : self.lans,\
660	}
661	self.element = { } # Current element being created
662	self.chars = "" # Last text seen
663
664	def end_element(self, name):
665	# After each sub element the contents is added to the current
666	# element or to the appropriate list.
667	if name == 'node':
668	self.nodes.append(self.element)
669	self.element = { }
670	elif name == 'lan':
671	self.lans.append(self.element)
672	self.element = { }
673	elif name in self.str_subelements:
674	self.element[name] = self.chars
675	self.chars = ""
676	elif name in self.int_subelements:
677	self.element[name] = int(self.chars)
678	self.chars = ""
679	elif name in self.float_subelements:
680	self.element[name] = float(self.chars)
681	self.chars = ""
682
683	def found_chars(self, data):
684	self.chars += data.rstrip()
685
686
687	tp = topo_parse();
688	parser = xml.parsers.expat.ParserCreate()
689	parser.EndElementHandler = tp.end_element
690	parser.CharacterDataHandler = tp.found_chars
691
692	parser.Parse(str)
693
694	return tp.topo
695
696
697	def genviz(self, topo):
698	"""
699	Generate the visualization the virtual topology
700	"""
701
702	neato = "/usr/local/bin/neato"
703	# These are used to parse neato output and to create the visualization
704	# file.
705	vis_re = re.compile('^\s"?([\w\-]+)"?\s+\[.pos="([\d\.]+),([\d\.]+)"')
706	vis_fmt = "<node><name>%s</name><x>%s</x><y>%s</y><type>" + \
707	"%s</type></node>"
708
709	try:
710	# Node names
711	nodes = [ n['vname'] for n in topo['node'] ]
712	topo_lans = topo['lan']
713	except KeyError, e:
714	raise service_error(service_error.internal, "Bad topology: %s" %e)
715
716	lans = { }
717	links = { }
718
719	# Walk through the virtual topology, organizing the connections into
720	# 2-node connections (links) and more-than-2-node connections (lans).
721	# When a lan is created, it's added to the list of nodes (there's a
722	# node in the visualization for the lan).
723	for l in topo_lans:
724	if links.has_key(l['vname']):
725	if len(links[l['vname']]) < 2:
726	links[l['vname']].append(l['vnode'])
727	else:
728	nodes.append(l['vname'])
729	lans[l['vname']] = links[l['vname']]
730	del links[l['vname']]
731	lans[l['vname']].append(l['vnode'])
732	elif lans.has_key(l['vname']):
733	lans[l['vname']].append(l['vnode'])
734	else:
735	links[l['vname']] = [ l['vnode'] ]
736
737
738	# Open up a temporary file for dot to turn into a visualization
739	try:
740	df, dotname = tempfile.mkstemp()
741	dotfile = os.fdopen(df, 'w')
742	except IOError:
743	raise service_error(service_error.internal,
744	"Failed to open file in genviz")
745
746	try:
747	dnull = open('/dev/null', 'w')
748	except IOError:
749	service_error(service_error.internal,
750	"Failed to open /dev/null in genviz")
751
752	# Generate a dot/neato input file from the links, nodes and lans
753	try:
754	print >>dotfile, "graph G {"
755	for n in nodes:
756	print >>dotfile, '\t"%s"' % n
757	for l in links.keys():
758	print >>dotfile, '\t"%s" -- "%s"' % tuple(links[l])
759	for l in lans.keys():
760	for n in lans[l]:
761	print >>dotfile, '\t "%s" -- "%s"' % (n,l)
762	print >>dotfile, "}"
763	dotfile.close()
764	except TypeError:
765	raise service_error(service_error.internal,
766	"Single endpoint link in vtopo")
767	except IOError:
768	raise service_error(service_error.internal, "Cannot write dot file")
769
770	# Use dot to create a visualization
771	dot = Popen([neato, '-Gstart=rand', '-Gepsilon=0.005', '-Gmaxiter=2000',
772	'-Gpack=true', dotname], stdout=PIPE, stderr=dnull,
773	close_fds=True)
774	dnull.close()
775
776	# Translate dot to vis format
777	vis_nodes = [ ]
778	vis = { 'node': vis_nodes }
779	for line in dot.stdout:
780	m = vis_re.match(line)
781	if m:
782	vn = m.group(1)
783	vis_node = {'name': vn, \
784	'x': float(m.group(2)),\
785	'y' : float(m.group(3)),\
786	}
787	if vn in links.keys() or vn in lans.keys():
788	vis_node['type'] = 'lan'
789	else:
790	vis_node['type'] = 'node'
791	vis_nodes.append(vis_node)
792	rv = dot.wait()
793
794	os.remove(dotname)
795	if rv == 0 : return vis
796	else: return None
797
798	def get_access(self, tb, nodes, tbparam, access_user, masters):
799	"""
800	Get access to testbed through fedd and set the parameters for that tb
801	"""
802	def get_export_project(svcs):
803	"""
804	Look through for the list of federated_service for this testbed
805	objects for a project_export service, and extract the project
806	parameter.
807	"""
808
809	pe = [s for s in svcs if s.name=='project_export']
810	if len(pe) == 1:
811	return pe[0].params.get('project', None)
812	elif len(pe) == 0:
813	return None
814	else:
815	raise service_error(service_error.req,
816	"More than one project export is not supported")
817
818	uri = self.tbmap.get(testbed_base(tb), None)
819	if not uri:
820	raise service_error(service_error.server_config,
821	"Unknown testbed: %s" % tb)
822
823	export_svcs = masters.get(tb,[])
824	import_svcs = [ s for m in masters.values() \
825	for s in m \
826	if tb in s.importers ]
827
828	export_project = get_export_project(export_svcs)
829
830	# Tweak search order so that if there are entries in access_user that
831	# have a project matching the export project, we try them first
832	if export_project:
833	access_sequence = [ (p, u) for p, u in access_user \
834	if p == export_project]
835	access_sequence.extend([(p, u) for p, u in access_user \
836	if p != export_project])
837	else:
838	access_sequence = access_user
839
840	for p, u in access_sequence:
841	self.log.debug(("[get_access] Attempting access from (%s, %s) " + \
842	"to %s") % ((p or "None"), u, uri))
843
844	if p:
845	# Request with user and project specified
846	req = {\
847	'destinationTestbed' : { 'uri' : uri },
848	'credential': [ "project: %s" % p, "user: %s" % u],
849	'allocID' : { 'localname': 'test' },
850	}
851	else:
852	# Request with only user specified
853	req = {\
854	'destinationTestbed' : { 'uri' : uri },
855	'credential': [ 'user: %s' % u ],
856	'allocID' : { 'localname': 'test' },
857	}
858
859	# Make the service request from the services we're importing and
860	# exporting. Keep track of the export request ids so we can
861	# collect the resulting info from the access response.
862	e_keys = { }
863	if import_svcs or export_svcs:
864	req['service'] = [ ]
865
866	for i, s in enumerate(import_svcs):
867	idx = 'import%d' % i
868	sr = {'id': idx, 'name': s.name, 'visibility': 'import' }
869	if s.params:
870	sr['fedAttr'] = [ { 'attribute': k, 'value': v } \
871	for k, v in s.params.items()]
872	req['service'].append(sr)
873
874	for i, s in enumerate(export_svcs):
875	idx = 'export%d' % i
876	e_keys[idx] = s
877	sr = {'id': idx, 'name': s.name, 'visibility': 'export' }
878	if s.params:
879	sr['fedAttr'] = [ { 'attribute': k, 'value': v }
880	for k, v in s.params.items()]
881	req['service'].append(sr)
882
883	# node resources if any
884	if nodes != None and len(nodes) > 0:
885	rnodes = [ ]
886	for n in nodes:
887	rn = { }
888	image, hw, count = n.split(":")
889	if image: rn['image'] = [ image ]
890	if hw: rn['hardware'] = [ hw ]
891	if count and int(count) >0 : rn['count'] = int(count)
892	rnodes.append(rn)
893	req['resources']= { }
894	req['resources']['node'] = rnodes
895
896	try:
897	if self.local_access.has_key(uri):
898	# Local access call
899	req = { 'RequestAccessRequestBody' : req }
900	r = self.local_access[uri].RequestAccess(req,
901	fedid(file=self.cert_file))
902	r = { 'RequestAccessResponseBody' : r }
903	else:
904	r = self.call_RequestAccess(uri, req,
905	self.cert_file, self.cert_pwd, self.trusted_certs)
906	except service_error, e:
907	if e.code == service_error.access:
908	self.log.debug("[get_access] Access denied")
909	r = None
910	continue
911	else:
912	raise e
913
914	if r.has_key('RequestAccessResponseBody'):
915	# Through to here we have a valid response, not a fault.
916	# Access denied is a fault, so something better or worse than
917	# access denied has happened.
918	r = r['RequestAccessResponseBody']
919	self.log.debug("[get_access] Access granted")
920	break
921	else:
922	raise service_error(service_error.protocol,
923	"Bad proxy response")
924
925	if not r:
926	raise service_error(service_error.access,
927	"Access denied by %s (%s)" % (tb, uri))
928
929	tbparam[tb] = {
930	"allocID" : r['allocID'],
931	"uri": uri,
932	}
933
934	# Collect the responses corresponding to the services this testbed
935	# exports. These will be the service requests that we will include in
936	# the start segment requests (with appropriate visibility values) to
937	# import and export the segments.
938	for s in r.get('service', []):
939	id = s.get('id', None)
940	if id and id in e_keys:
941	e_keys[id].reqs.append(s)
942
943	# Add attributes to parameter space. We don't allow attributes to
944	# overlay any parameters already installed.
945	for a in r.get('fedAttr', []):
946	try:
947	if a['attribute'] and \
948	isinstance(a['attribute'], basestring)\
949	and not tbparam[tb].has_key(a['attribute'].lower()):
950	tbparam[tb][a['attribute'].lower()] = a['value']
951	except KeyError:
952	self.log.error("Bad attribute in response: %s" % a)
953
954	def release_access(self, tb, aid, uri=None):
955	"""
956	Release access to testbed through fedd
957	"""
958
959	if not uri:
960	uri = self.tbmap.get(tb, None)
961	if not uri:
962	raise service_error(service_error.server_config,
963	"Unknown testbed: %s" % tb)
964
965	if self.local_access.has_key(uri):
966	resp = self.local_access[uri].ReleaseAccess(\
967	{ 'ReleaseAccessRequestBody' : {'allocID': aid},},
968	fedid(file=self.cert_file))
969	resp = { 'ReleaseAccessResponseBody': resp }
970	else:
971	resp = self.call_ReleaseAccess(uri, {'allocID': aid},
972	self.cert_file, self.cert_pwd, self.trusted_certs)
973
974	# better error coding
975
976	def remote_ns2topdl(self, uri, desc):
977
978	req = {
979	'description' : { 'ns2description': desc },
980	}
981
982	r = self.call_Ns2Topdl(uri, req, self.cert_file, self.cert_pwd,
983	self.trusted_certs)
984
985	if r.has_key('Ns2TopdlResponseBody'):
986	r = r['Ns2TopdlResponseBody']
987	ed = r.get('experimentdescription', None)
988	if ed.has_key('topdldescription'):
989	return topdl.Topology(**ed['topdldescription'])
990	else:
991	raise service_error(service_error.protocol,
992	"Bad splitter response (no output)")
993	else:
994	raise service_error(service_error.protocol, "Bad splitter response")
995
996	class start_segment:
997	def __init__(self, debug=False, log=None, testbed="", cert_file=None,
998	cert_pwd=None, trusted_certs=None, caller=None,
999	log_collector=None):
1000	self.log = log
1001	self.debug = debug
1002	self.cert_file = cert_file
1003	self.cert_pwd = cert_pwd
1004	self.trusted_certs = None
1005	self.caller = caller
1006	self.testbed = testbed
1007	self.log_collector = log_collector
1008	self.response = None
1009	self.node = { }
1010
1011	def make_map(self, resp):
1012	if 'segmentdescription' in resp and \
1013	'topdldescription' in resp['segmentdescription']:
1014	top = topdl.Topology(\
1015	**resp['segmentdescription']['topdldescription'])
1016	for e in [e for e in top.elements \
1017	if isinstance(e, topdl.Computer)]:
1018	hn = e.get_attribute('hostname')
1019	if hn:
1020	for n in e.name:
1021	self.node[n] = hn
1022
1023	def __call__(self, uri, aid, topo, masters, attrs=None, connInfo=None):
1024	req = {
1025	'allocID': { 'fedid' : aid },
1026	'segmentdescription': {
1027	'topdldescription': topo.to_dict(),
1028	},
1029	}
1030
1031	if connInfo:
1032	req['connection'] = connInfo
1033
1034	import_svcs = [ s for m in masters.values() \
1035	for s in m if self.testbed in s.importers]
1036
1037	if import_svcs or self.testbed in masters:
1038	req['service'] = []
1039
1040	for s in import_svcs:
1041	for r in s.reqs:
1042	sr = copy.deepcopy(r)
1043	sr['visibility'] = 'import';
1044	req['service'].append(sr)
1045
1046	for s in masters.get(self.testbed, []):
1047	for r in s.reqs:
1048	sr = copy.deepcopy(r)
1049	sr['visibility'] = 'export';
1050	req['service'].append(sr)
1051
1052	if attrs:
1053	req['fedAttr'] = attrs
1054
1055	try:
1056	self.log.debug("Calling StartSegment at %s " % uri)
1057	r = self.caller(uri, req, self.cert_file, self.cert_pwd,
1058	self.trusted_certs)
1059	if r.has_key('StartSegmentResponseBody'):
1060	lval = r['StartSegmentResponseBody'].get('allocationLog',
1061	None)
1062	if lval and self.log_collector:
1063	for line in lval.splitlines(True):
1064	self.log_collector.write(line)
1065	self.make_map(r['StartSegmentResponseBody'])
1066	self.response = r
1067	else:
1068	raise service_error(service_error.internal,
1069	"Bad response!?: %s" %r)
1070	return True
1071	except service_error, e:
1072	self.log.error("Start segment failed on %s: %s" % \
1073	(self.testbed, e))
1074	return False
1075
1076
1077
1078	class terminate_segment:
1079	def __init__(self, debug=False, log=None, testbed="", cert_file=None,
1080	cert_pwd=None, trusted_certs=None, caller=None):
1081	self.log = log
1082	self.debug = debug
1083	self.cert_file = cert_file
1084	self.cert_pwd = cert_pwd
1085	self.trusted_certs = None
1086	self.caller = caller
1087	self.testbed = testbed
1088
1089	def __call__(self, uri, aid ):
1090	req = {
1091	'allocID': aid ,
1092	}
1093	try:
1094	r = self.caller(uri, req, self.cert_file, self.cert_pwd,
1095	self.trusted_certs)
1096	return True
1097	except service_error, e:
1098	self.log.error("Terminate segment failed on %s: %s" % \
1099	(self.testbed, e))
1100	return False
1101
1102
1103	def allocate_resources(self, allocated, masters, eid, expid,
1104	tbparams, top, topo, tmpdir, alloc_log=None, log_collector=None,
1105	attrs=None, connInfo={}):
1106
1107	started = { } # Testbeds where a sub-experiment started
1108	# successfully
1109
1110	# XXX
1111	fail_soft = False
1112
1113	log = alloc_log or self.log
1114
1115	thread_pool = self.thread_pool(self.nthreads)
1116	threads = [ ]
1117	starters = [ ]
1118
1119	for tb in allocated.keys():
1120	# Create and start a thread to start the segment, and save it
1121	# to get the return value later
1122	tb_attrs = copy.copy(attrs)
1123	thread_pool.wait_for_slot()
1124	uri = tbparams[tb].get('uri', \
1125	self.tbmap.get(testbed_base(tb), None))
1126	base, suffix = split_testbed(tb)
1127	if suffix:
1128	tb_attrs.append({'attribute': 'experiment_name',
1129	'value': "%s_%s" % (eid, suffix)})
1130	else:
1131	tb_attrs.append({'attribute': 'experiment_name', 'value': eid})
1132	if not uri:
1133	raise service_error(service_error.internal,
1134	"Unknown testbed %s !?" % tb)
1135
1136	if tbparams[tb].has_key('allocID') and \
1137	tbparams[tb]['allocID'].has_key('fedid'):
1138	aid = tbparams[tb]['allocID']['fedid']
1139	else:
1140	raise service_error(service_error.internal,
1141	"No alloc id for testbed %s !?" % tb)
1142
1143	s = self.start_segment(log=log, debug=self.debug,
1144	testbed=tb, cert_file=self.cert_file,
1145	cert_pwd=self.cert_pwd, trusted_certs=self.trusted_certs,
1146	caller=self.call_StartSegment,
1147	log_collector=log_collector)
1148	starters.append(s)
1149	t = self.pooled_thread(\
1150	target=s, name=tb,
1151	args=(uri, aid, topo[tb], masters, tb_attrs, connInfo[tb]),
1152	pdata=thread_pool, trace_file=self.trace_file)
1153	threads.append(t)
1154	t.start()
1155
1156	# Wait until all finish (keep pinging the log, though)
1157	mins = 0
1158	revoked = False
1159	while not thread_pool.wait_for_all_done(60.0):
1160	mins += 1
1161	alloc_log.info("Waiting for sub threads (it has been %d mins)" \
1162	% mins)
1163	if not revoked and \
1164	len([ t.getName() for t in threads if t.rv == False]) > 0:
1165	# a testbed has failed. Revoke this experiment's
1166	# synchronizarion values so that sub experiments will not
1167	# deadlock waiting for synchronization that will never happen
1168	self.log.info("A subexperiment has failed to swap in, " + \
1169	"revoking synch keys")
1170	var_key = "fedid:%s" % expid
1171	for k in self.synch_store.all_keys():
1172	if len(k) > 45 and k[0:46] == var_key:
1173	self.synch_store.revoke_key(k)
1174	revoked = True
1175
1176	failed = [ t.getName() for t in threads if not t.rv ]
1177	succeeded = [tb for tb in allocated.keys() if tb not in failed]
1178
1179	# If one failed clean up, unless fail_soft is set
1180	if failed:
1181	if not fail_soft:
1182	thread_pool.clear()
1183	for tb in succeeded:
1184	# Create and start a thread to stop the segment
1185	thread_pool.wait_for_slot()
1186	uri = tbparams[tb]['uri']
1187	t = self.pooled_thread(\
1188	target=self.terminate_segment(log=log,
1189	testbed=tb,
1190	cert_file=self.cert_file,
1191	cert_pwd=self.cert_pwd,
1192	trusted_certs=self.trusted_certs,
1193	caller=self.call_TerminateSegment),
1194	args=(uri, tbparams[tb]['federant']['allocID']),
1195	name=tb,
1196	pdata=thread_pool, trace_file=self.trace_file)
1197	t.start()
1198	# Wait until all finish (if any are being stopped)
1199	if succeeded:
1200	thread_pool.wait_for_all_done()
1201
1202	# release the allocations
1203	for tb in tbparams.keys():
1204	self.release_access(tb, tbparams[tb]['allocID'],
1205	tbparams[tb].get('uri', None))
1206	# Remove the placeholder
1207	self.state_lock.acquire()
1208	self.state[eid]['experimentStatus'] = 'failed'
1209	if self.state_filename: self.write_state()
1210	self.state_lock.release()
1211
1212	log.error("Swap in failed on %s" % ",".join(failed))
1213	return
1214	else:
1215	# Walk through the successes and gather the virtual to physical
1216	# mapping.
1217	node = { }
1218	for s in starters:
1219	node.update(s.node)
1220	# Assigng the mapping as a hostname attribute
1221	for e in [ e for e in top.elements \
1222	if isinstance(e, topdl.Computer)]:
1223	for n in e.name:
1224	if n in node:
1225	e.set_attribute('hostname', node[n])
1226	log.info("[start_segment]: Experiment %s active" % eid)
1227
1228
1229	# Walk up tmpdir, deleting as we go
1230	if self.cleanup:
1231	log.debug("[start_experiment]: removing %s" % tmpdir)
1232	for path, dirs, files in os.walk(tmpdir, topdown=False):
1233	for f in files:
1234	os.remove(os.path.join(path, f))
1235	for d in dirs:
1236	os.rmdir(os.path.join(path, d))
1237	os.rmdir(tmpdir)
1238	else:
1239	log.debug("[start_experiment]: not removing %s" % tmpdir)
1240
1241	# Insert the experiment into our state and update the disk copy.
1242	self.state_lock.acquire()
1243	self.state[expid]['experimentStatus'] = 'active'
1244	self.state[eid] = self.state[expid]
1245	self.state[eid]['experimentdescription']['topdldescription'] = \
1246	top.to_dict()
1247	if self.state_filename: self.write_state()
1248	self.state_lock.release()
1249	return
1250
1251
1252	def add_kit(self, e, kit):
1253	"""
1254	Add a Software object created from the list of (install, location)
1255	tuples passed as kit to the software attribute of an object e. We
1256	do this enough to break out the code, but it's kind of a hack to
1257	avoid changing the old tuple rep.
1258	"""
1259
1260	s = [ topdl.Software(install=i, location=l) for i, l in kit]
1261
1262	if isinstance(e.software, list): e.software.extend(s)
1263	else: e.software = s
1264
1265
1266	def create_experiment_state(self, fid, req, expid, expcert,
1267	state='starting'):
1268	"""
1269	Create the initial entry in the experiment's state. The expid and
1270	expcert are the experiment's fedid and certifacte that represents that
1271	ID, which are installed in the experiment state. If the request
1272	includes a suggested local name that is used if possible. If the local
1273	name is already taken by an experiment owned by this user that has
1274	failed, it is overwritten. Otherwise new letters are added until a
1275	valid localname is found. The generated local name is returned.
1276	"""
1277
1278	if req.has_key('experimentID') and \
1279	req['experimentID'].has_key('localname'):
1280	overwrite = False
1281	eid = req['experimentID']['localname']
1282	# If there's an old failed experiment here with the same local name
1283	# and accessible by this user, we'll overwrite it, otherwise we'll
1284	# fall through and do the collision avoidance.
1285	old_expid = self.get_experiment_fedid(eid)
1286	if old_expid and self.check_experiment_access(fid, old_expid):
1287	self.state_lock.acquire()
1288	status = self.state[eid].get('experimentStatus', None)
1289	if status and status == 'failed':
1290	# remove the old access attribute
1291	self.auth.unset_attribute(fid, old_expid)
1292	overwrite = True
1293	del self.state[eid]
1294	del self.state[old_expid]
1295	self.state_lock.release()
1296	self.state_lock.acquire()
1297	while (self.state.has_key(eid) and not overwrite):
1298	eid += random.choice(string.ascii_letters)
1299	# Initial state
1300	self.state[eid] = {
1301	'experimentID' : \
1302	[ { 'localname' : eid }, {'fedid': expid } ],
1303	'experimentStatus': state,
1304	'experimentAccess': { 'X509' : expcert },
1305	'owner': fid,
1306	'log' : [],
1307	}
1308	self.state[expid] = self.state[eid]
1309	if self.state_filename: self.write_state()
1310	self.state_lock.release()
1311	else:
1312	eid = self.exp_stem
1313	for i in range(0,5):
1314	eid += random.choice(string.ascii_letters)
1315	self.state_lock.acquire()
1316	while (self.state.has_key(eid)):
1317	eid = self.exp_stem
1318	for i in range(0,5):
1319	eid += random.choice(string.ascii_letters)
1320	# Initial state
1321	self.state[eid] = {
1322	'experimentID' : \
1323	[ { 'localname' : eid }, {'fedid': expid } ],
1324	'experimentStatus': state,
1325	'experimentAccess': { 'X509' : expcert },
1326	'owner': fid,
1327	'log' : [],
1328	}
1329	self.state[expid] = self.state[eid]
1330	if self.state_filename: self.write_state()
1331	self.state_lock.release()
1332
1333	return eid
1334
1335
1336	def allocate_ips_to_topo(self, top):
1337	"""
1338	Add an ip4_address attribute to all the hosts in the topology, based on
1339	the shared substrates on which they sit. An /etc/hosts file is also
1340	created and returned as a list of hostfiles entries. We also return
1341	the allocator, because we may need to allocate IPs to portals
1342	(specifically DRAGON portals).
1343	"""
1344	subs = sorted(top.substrates,
1345	cmp=lambda x,y: cmp(len(x.interfaces), len(y.interfaces)),
1346	reverse=True)
1347	ips = ip_allocator(int(ip_addr("10.0.0.0")), 2 **24)
1348	ifs = { }
1349	hosts = [ ]
1350
1351	for idx, s in enumerate(subs):
1352	net_size = len(s.interfaces)+2
1353
1354	a = ips.allocate(net_size)
1355	if a :
1356	base, num = a
1357	if num < net_size:
1358	raise service_error(service_error.internal,
1359	"Allocator returned wrong number of IPs??")
1360	else:
1361	raise service_error(service_error.req,
1362	"Cannot allocate IP addresses")
1363	mask = ips.min_alloc
1364	while mask < net_size:
1365	mask *= 2
1366
1367	netmask = ((2**32-1) ^ (mask-1))
1368
1369	base += 1
1370	for i in s.interfaces:
1371	i.attribute.append(
1372	topdl.Attribute('ip4_address',
1373	"%s" % ip_addr(base)))
1374	i.attribute.append(
1375	topdl.Attribute('ip4_netmask',
1376	"%s" % ip_addr(int(netmask))))
1377
1378	hname = i.element.name[0]
1379	if ifs.has_key(hname):
1380	hosts.append("%s\t%s-%s %s-%d" % \
1381	(ip_addr(base), hname, s.name, hname,
1382	ifs[hname]))
1383	else:
1384	ifs[hname] = 0
1385	hosts.append("%s\t%s-%s %s-%d %s" % \
1386	(ip_addr(base), hname, s.name, hname,
1387	ifs[hname], hname))
1388
1389	ifs[hname] += 1
1390	base += 1
1391	return hosts, ips
1392
1393	def get_access_to_testbeds(self, testbeds, access_user, allocated,
1394	tbparams, masters):
1395	"""
1396	Request access to the various testbeds required for this instantiation
1397	(passed in as testbeds). User, access_user, expoert_project and master
1398	are used to construct the correct requests. Per-testbed parameters are
1399	returned in tbparams.
1400	"""
1401	for tb in testbeds:
1402	self.get_access(tb, None, tbparams, access_user, masters)
1403	allocated[tb] = 1
1404
1405	def split_topology(self, top, topo, testbeds):
1406	"""
1407	Create the sub-topologies that are needed for experiment instantiation.
1408	"""
1409	for tb in testbeds:
1410	topo[tb] = top.clone()
1411	# copy in for loop allows deletions from the original
1412	for e in [ e for e in topo[tb].elements]:
1413	etb = e.get_attribute('testbed')
1414	# NB: elements without a testbed attribute won't appear in any
1415	# sub topologies.
1416	if not etb or etb != tb:
1417	for i in e.interface:
1418	for s in i.subs:
1419	try:
1420	s.interfaces.remove(i)
1421	except ValueError:
1422	raise service_error(service_error.internal,
1423	"Can't remove interface??")
1424	topo[tb].elements.remove(e)
1425	topo[tb].make_indices()
1426
1427	def wrangle_software(self, expid, top, topo, tbparams):
1428	"""
1429	Copy software out to the repository directory, allocate permissions and
1430	rewrite the segment topologies to look for the software in local
1431	places.
1432	"""
1433
1434	# Copy the rpms and tarfiles to a distribution directory from
1435	# which the federants can retrieve them
1436	linkpath = "%s/software" % expid
1437	softdir ="%s/%s" % ( self.repodir, linkpath)
1438	softmap = { }
1439	# These are in a list of tuples format (each kit). This comprehension
1440	# unwraps them into a single list of tuples that initilaizes the set of
1441	# tuples.
1442	pkgs = set([ t for l in [self.fedkit, self.gatewaykit] \
1443	for p, t in l ])
1444	pkgs.update([x.location for e in top.elements \
1445	for x in e.software])
1446	try:
1447	os.makedirs(softdir)
1448	except IOError, e:
1449	raise service_error(
1450	"Cannot create software directory: %s" % e)
1451	# The actual copying. Everything's converted into a url for copying.
1452	for pkg in pkgs:
1453	loc = pkg
1454
1455	scheme, host, path = urlparse(loc)[0:3]
1456	dest = os.path.basename(path)
1457	if not scheme:
1458	if not loc.startswith('/'):
1459	loc = "/%s" % loc
1460	loc = "file://%s" %loc
1461	try:
1462	u = urlopen(loc)
1463	except Exception, e:
1464	raise service_error(service_error.req,
1465	"Cannot open %s: %s" % (loc, e))
1466	try:
1467	f = open("%s/%s" % (softdir, dest) , "w")
1468	self.log.debug("Writing %s/%s" % (softdir,dest) )
1469	data = u.read(4096)
1470	while data:
1471	f.write(data)
1472	data = u.read(4096)
1473	f.close()
1474	u.close()
1475	except Exception, e:
1476	raise service_error(service_error.internal,
1477	"Could not copy %s: %s" % (loc, e))
1478	path = re.sub("/tmp", "", linkpath)
1479	# XXX
1480	softmap[pkg] = \
1481	"%s/%s/%s" %\
1482	( self.repo_url, path, dest)
1483
1484	# Allow the individual segments to access the software.
1485	for tb in tbparams.keys():
1486	self.auth.set_attribute(tbparams[tb]['allocID']['fedid'],
1487	"/%s/%s" % ( path, dest))
1488
1489	# Convert the software locations in the segments into the local
1490	# copies on this host
1491	for soft in [ s for tb in topo.values() \
1492	for e in tb.elements \
1493	if getattr(e, 'software', False) \
1494	for s in e.software ]:
1495	if softmap.has_key(soft.location):
1496	soft.location = softmap[soft.location]
1497
1498
1499	def new_experiment(self, req, fid):
1500	"""
1501	The external interface to empty initial experiment creation called from
1502	the dispatcher.
1503
1504	Creates a working directory, splits the incoming description using the
1505	splitter script and parses out the avrious subsections using the
1506	lcasses above. Once each sub-experiment is created, use pooled threads
1507	to instantiate them and start it all up.
1508	"""
1509	if not self.auth.check_attribute(fid, 'new'):
1510	raise service_error(service_error.access, "New access denied")
1511
1512	try:
1513	tmpdir = tempfile.mkdtemp(prefix="split-")
1514	except IOError:
1515	raise service_error(service_error.internal, "Cannot create tmp dir")
1516
1517	try:
1518	access_user = self.accessdb[fid]
1519	except KeyError:
1520	raise service_error(service_error.internal,
1521	"Access map and authorizer out of sync in " + \
1522	"new_experiment for fedid %s" % fid)
1523
1524	pid = "dummy"
1525	gid = "dummy"
1526
1527	req = req.get('NewRequestBody', None)
1528	if not req:
1529	raise service_error(service_error.req,
1530	"Bad request format (no NewRequestBody)")
1531
1532	# Generate an ID for the experiment (slice) and a certificate that the
1533	# allocator can use to prove they own it. We'll ship it back through
1534	# the encrypted connection.
1535	(expid, expcert) = generate_fedid("test", dir=tmpdir, log=self.log)
1536
1537	#now we're done with the tmpdir, and it should be empty
1538	if self.cleanup:
1539	self.log.debug("[new_experiment]: removing %s" % tmpdir)
1540	os.rmdir(tmpdir)
1541	else:
1542	self.log.debug("[new_experiment]: not removing %s" % tmpdir)
1543
1544	eid = self.create_experiment_state(fid, req, expid, expcert,
1545	state='empty')
1546
1547	# Let users touch the state
1548	self.auth.set_attribute(fid, expid)
1549	self.auth.set_attribute(expid, expid)
1550	# Override fedids can manipulate state as well
1551	for o in self.overrides:
1552	self.auth.set_attribute(o, expid)
1553
1554	rv = {
1555	'experimentID': [
1556	{'localname' : eid }, { 'fedid': copy.copy(expid) }
1557	],
1558	'experimentStatus': 'empty',
1559	'experimentAccess': { 'X509' : expcert }
1560	}
1561
1562	return rv
1563
1564	def create_experiment(self, req, fid):
1565	"""
1566	The external interface to experiment creation called from the
1567	dispatcher.
1568
1569	Creates a working directory, splits the incoming description using the
1570	splitter script and parses out the various subsections using the
1571	classes above. Once each sub-experiment is created, use pooled threads
1572	to instantiate them and start it all up.
1573	"""
1574
1575	req = req.get('CreateRequestBody', None)
1576	if not req:
1577	raise service_error(service_error.req,
1578	"Bad request format (no CreateRequestBody)")
1579
1580	# Get the experiment access
1581	exp = req.get('experimentID', None)
1582	if exp:
1583	if exp.has_key('fedid'):
1584	key = exp['fedid']
1585	expid = key
1586	eid = None
1587	elif exp.has_key('localname'):
1588	key = exp['localname']
1589	eid = key
1590	expid = None
1591	else:
1592	raise service_error(service_error.req, "Unknown lookup type")
1593	else:
1594	raise service_error(service_error.req, "No request?")
1595
1596	self.check_experiment_access(fid, key)
1597
1598	try:
1599	tmpdir = tempfile.mkdtemp(prefix="split-")
1600	os.mkdir(tmpdir+"/keys")
1601	except IOError:
1602	raise service_error(service_error.internal, "Cannot create tmp dir")
1603
1604	gw_pubkey_base = "fed.%s.pub" % self.ssh_type
1605	gw_secretkey_base = "fed.%s" % self.ssh_type
1606	gw_pubkey = tmpdir + "/keys/" + gw_pubkey_base
1607	gw_secretkey = tmpdir + "/keys/" + gw_secretkey_base
1608	tclfile = tmpdir + "/experiment.tcl"
1609	tbparams = { }
1610	try:
1611	access_user = self.accessdb[fid]
1612	except KeyError:
1613	raise service_error(service_error.internal,
1614	"Access map and authorizer out of sync in " + \
1615	"create_experiment for fedid %s" % fid)
1616
1617	pid = "dummy"
1618	gid = "dummy"
1619
1620	# The tcl parser needs to read a file so put the content into that file
1621	descr=req.get('experimentdescription', None)
1622	if descr:
1623	file_content=descr.get('ns2description', None)
1624	if file_content:
1625	try:
1626	f = open(tclfile, 'w')
1627	f.write(file_content)
1628	f.close()
1629	except IOError:
1630	raise service_error(service_error.internal,
1631	"Cannot write temp experiment description")
1632	else:
1633	raise service_error(service_error.req,
1634	"Only ns2descriptions supported")
1635	else:
1636	raise service_error(service_error.req, "No experiment description")
1637
1638	self.state_lock.acquire()
1639	if self.state.has_key(key):
1640	self.state[key]['experimentStatus'] = "starting"
1641	for e in self.state[key].get('experimentID',[]):
1642	if not expid and e.has_key('fedid'):
1643	expid = e['fedid']
1644	elif not eid and e.has_key('localname'):
1645	eid = e['localname']
1646	self.state_lock.release()
1647
1648	if not (eid and expid):
1649	raise service_error(service_error.internal,
1650	"Cannot find local experiment info!?")
1651
1652	try:
1653	# This catches exceptions to clear the placeholder if necessary
1654	try:
1655	self.generate_ssh_keys(gw_secretkey, self.ssh_type)
1656	except ValueError:
1657	raise service_error(service_error.server_config,
1658	"Bad key type (%s)" % self.ssh_type)
1659
1660	# Copy the service request
1661	tb_services = [ s for s in req.get('service',[]) ]
1662	# Translate to topdl
1663	if self.splitter_url:
1664	self.log.debug("Calling remote topdl translator at %s" % \
1665	self.splitter_url)
1666	top = self.remote_ns2topdl(self.splitter_url, file_content)
1667	else:
1668	tclcmd = [self.tclsh, self.tcl_splitter, '-t', '-x',
1669	str(self.muxmax), '-m', 'dummy']
1670
1671	tclcmd.extend([pid, gid, eid, tclfile])
1672
1673	self.log.debug("running local splitter %s", " ".join(tclcmd))
1674	# This is just fantastic. As a side effect the parser copies
1675	# tb_compat.tcl into the current directory, so that directory
1676	# must be writable by the fedd user. Doing this in the
1677	# temporary subdir ensures this is the case.
1678	tclparser = Popen(tclcmd, stdout=PIPE, close_fds=True,
1679	cwd=tmpdir)
1680	split_data = tclparser.stdout
1681
1682	top = topdl.topology_from_xml(file=split_data, top="experiment")
1683
1684	hosts, ip_allocator = self.allocate_ips_to_topo(top)
1685	# Find the testbeds to look up
1686	testbeds = set([ a.value for e in top.elements \
1687	for a in e.attribute \
1688	if a.attribute == 'testbed'])
1689
1690	masters = { } # testbeds exporting services
1691	for s in tb_services:
1692	# If this is a project_export request with the imports field
1693	# blank, fill it in.
1694	if s.get('name', '') == 'project_export':
1695	if 'import' not in s or len(s['import']) == 0:
1696	s['import'] = [ tb for tb in testbeds \
1697	if tb not in s.get('export',[])]
1698	# Add the service to masters
1699	for tb in s.get('export', []):
1700	if s.get('name', None):
1701	if tb not in masters:
1702	masters[tb] = [ ]
1703
1704	params = { }
1705	if 'fedAttr' in s:
1706	for a in s['fedAttr']:
1707	params[a.get('attribute', '')] = \
1708	a.get('value','')
1709
1710	masters[tb].append(federated_service(name=s['name'],
1711	exporter=tb, importers=s.get('import',[]),
1712	params=params))
1713	else:
1714	self.log.error('Testbed service does not have name " + \
1715	"and importers')
1716
1717
1718	allocated = { } # Testbeds we can access
1719	topo ={ } # Sub topologies
1720	connInfo = { } # Connection information
1721	self.get_access_to_testbeds(testbeds, access_user, allocated,
1722	tbparams, masters)
1723
1724	self.split_topology(top, topo, testbeds)
1725
1726	# Copy configuration files into the remote file store
1727	# The config urlpath
1728	configpath = "/%s/config" % expid
1729	# The config file system location
1730	configdir ="%s%s" % ( self.repodir, configpath)
1731	try:
1732	os.makedirs(configdir)
1733	except EnvironmentError, e:
1734	raise service_error(service_error.internal,
1735	"Cannot create config directory: %s" % e)
1736	try:
1737	f = open("%s/hosts" % configdir, "w")
1738	f.write('\n'.join(hosts))
1739	f.close()
1740	except IOError, e:
1741	raise service_error(service_error.internal,
1742	"Cannot write hosts file: %s" % e)
1743	try:
1744	copy_file("%s" % gw_pubkey, "%s/%s" % \
1745	(configdir, gw_pubkey_base))
1746	copy_file("%s" % gw_secretkey, "%s/%s" % \
1747	(configdir, gw_secretkey_base))
1748	except IOError, e:
1749	raise service_error(service_error.internal,
1750	"Cannot copy keyfiles: %s" % e)
1751
1752	# Allow the individual testbeds to access the configuration files.
1753	for tb in tbparams.keys():
1754	asignee = tbparams[tb]['allocID']['fedid']
1755	for f in ("hosts", gw_secretkey_base, gw_pubkey_base):
1756	self.auth.set_attribute(asignee, "%s/%s" % (configpath, f))
1757
1758	part = experiment_partition(self.auth, self.store_url, self.tbmap,
1759	self.muxmax)
1760	part.add_portals(top, topo, eid, masters, tbparams, ip_allocator,
1761	connInfo, expid)
1762	# Now get access to the dynamic testbeds (those added above)
1763	for tb in [ t for t in topo if t not in allocated]:
1764	self.get_access(tb, None, tbparams, access_user, masters)
1765	allocated[tb] = 1
1766	store_keys = topo[tb].get_attribute('store_keys')
1767	# Give the testbed access to keys it exports or imports
1768	if store_keys:
1769	for sk in store_keys.split(" "):
1770	self.auth.set_attribute(\
1771	tbparams[tb]['allocID']['fedid'], sk)
1772
1773	self.wrangle_software(expid, top, topo, tbparams)
1774
1775	vtopo = topdl.topology_to_vtopo(top)
1776	vis = self.genviz(vtopo)
1777
1778	# save federant information
1779	for k in allocated.keys():
1780	tbparams[k]['federant'] = {
1781	'name': [ { 'localname' : eid} ],
1782	'allocID' : tbparams[k]['allocID'],
1783	'uri': tbparams[k]['uri'],
1784	}
1785
1786	self.state_lock.acquire()
1787	self.state[eid]['vtopo'] = vtopo
1788	self.state[eid]['vis'] = vis
1789	self.state[eid]['experimentdescription'] = \
1790	{ 'topdldescription': top.clone() }
1791	self.state[expid]['federant'] = \
1792	[ tbparams[tb]['federant'] for tb in tbparams.keys() \
1793	if tbparams[tb].has_key('federant') ]
1794	if self.state_filename:
1795	self.write_state()
1796	self.state_lock.release()
1797	except service_error, e:
1798	# If something goes wrong in the parse (usually an access error)
1799	# clear the placeholder state. From here on out the code delays
1800	# exceptions. Failing at this point returns a fault to the remote
1801	# caller.
1802
1803	self.state_lock.acquire()
1804	del self.state[eid]
1805	del self.state[expid]
1806	if self.state_filename: self.write_state()
1807	self.state_lock.release()
1808	raise e
1809
1810
1811	# Start the background swapper and return the starting state. From
1812	# here on out, the state will stick around a while.
1813
1814	# Let users touch the state
1815	self.auth.set_attribute(fid, expid)
1816	self.auth.set_attribute(expid, expid)
1817	# Override fedids can manipulate state as well
1818	for o in self.overrides:
1819	self.auth.set_attribute(o, expid)
1820
1821	# Create a logger that logs to the experiment's state object as well as
1822	# to the main log file.
1823	alloc_log = logging.getLogger('fedd.experiment_control.%s' % eid)
1824	alloc_collector = self.list_log(self.state[eid]['log'])
1825	h = logging.StreamHandler(alloc_collector)
1826	# XXX: there should be a global one of these rather than repeating the
1827	# code.
1828	h.setFormatter(logging.Formatter("%(asctime)s %(name)s %(message)s",
1829	'%d %b %y %H:%M:%S'))
1830	alloc_log.addHandler(h)
1831
1832	attrs = [
1833	{
1834	'attribute': 'ssh_pubkey',
1835	'value': '%s/%s/config/%s' % \
1836	(self.repo_url, expid, gw_pubkey_base)
1837	},
1838	{
1839	'attribute': 'ssh_secretkey',
1840	'value': '%s/%s/config/%s' % \
1841	(self.repo_url, expid, gw_secretkey_base)
1842	},
1843	{
1844	'attribute': 'hosts',
1845	'value': '%s/%s/config/hosts' % \
1846	(self.repo_url, expid)
1847	},
1848	]
1849
1850	# transit and disconnected testbeds may not have a connInfo entry.
1851	# Fill in the blanks.
1852	for t in allocated.keys():
1853	if not connInfo.has_key(t):
1854	connInfo[t] = { }
1855
1856	# Start a thread to do the resource allocation
1857	t = Thread(target=self.allocate_resources,
1858	args=(allocated, masters, eid, expid, tbparams,
1859	top, topo, tmpdir, alloc_log, alloc_collector, attrs,
1860	connInfo),
1861	name=eid)
1862	t.start()
1863
1864	rv = {
1865	'experimentID': [
1866	{'localname' : eid }, { 'fedid': copy.copy(expid) }
1867	],
1868	'experimentStatus': 'starting',
1869	}
1870
1871	return rv
1872
1873	def get_experiment_fedid(self, key):
1874	"""
1875	find the fedid associated with the localname key in the state database.
1876	"""
1877
1878	rv = None
1879	self.state_lock.acquire()
1880	if self.state.has_key(key):
1881	if isinstance(self.state[key], dict):
1882	try:
1883	kl = [ f['fedid'] for f in \
1884	self.state[key]['experimentID']\
1885	if f.has_key('fedid') ]
1886	except KeyError:
1887	self.state_lock.release()
1888	raise service_error(service_error.internal,
1889	"No fedid for experiment %s when getting "+\
1890	"fedid(!?)" % key)
1891	if len(kl) == 1:
1892	rv = kl[0]
1893	else:
1894	self.state_lock.release()
1895	raise service_error(service_error.internal,
1896	"multiple fedids for experiment %s when " +\
1897	"getting fedid(!?)" % key)
1898	else:
1899	self.state_lock.release()
1900	raise service_error(service_error.internal,
1901	"Unexpected state for %s" % key)
1902	self.state_lock.release()
1903	return rv
1904
1905	def check_experiment_access(self, fid, key):
1906	"""
1907	Confirm that the fid has access to the experiment. Though a request
1908	may be made in terms of a local name, the access attribute is always
1909	the experiment's fedid.
1910	"""
1911	if not isinstance(key, fedid):
1912	key = self.get_experiment_fedid(key)
1913
1914	if self.auth.check_attribute(fid, key):
1915	return True
1916	else:
1917	raise service_error(service_error.access, "Access Denied")
1918
1919
1920	def get_handler(self, path, fid):
1921	self.log.info("Get handler %s %s" % (path, fid))
1922	if self.auth.check_attribute(fid, path):
1923	return ("%s/%s" % (self.repodir, path), "application/binary")
1924	else:
1925	return (None, None)
1926
1927	def get_vtopo(self, req, fid):
1928	"""
1929	Return the stored virtual topology for this experiment
1930	"""
1931	rv = None
1932	state = None
1933
1934	req = req.get('VtopoRequestBody', None)
1935	if not req:
1936	raise service_error(service_error.req,
1937	"Bad request format (no VtopoRequestBody)")
1938	exp = req.get('experiment', None)
1939	if exp:
1940	if exp.has_key('fedid'):
1941	key = exp['fedid']
1942	keytype = "fedid"
1943	elif exp.has_key('localname'):
1944	key = exp['localname']
1945	keytype = "localname"
1946	else:
1947	raise service_error(service_error.req, "Unknown lookup type")
1948	else:
1949	raise service_error(service_error.req, "No request?")
1950
1951	self.check_experiment_access(fid, key)
1952
1953	self.state_lock.acquire()
1954	if self.state.has_key(key):
1955	if self.state[key].has_key('vtopo'):
1956	rv = { 'experiment' : {keytype: key },\
1957	'vtopo': self.state[key]['vtopo'],\
1958	}
1959	else:
1960	state = self.state[key]['experimentStatus']
1961	self.state_lock.release()
1962
1963	if rv: return rv
1964	else:
1965	if state:
1966	raise service_error(service_error.partial,
1967	"Not ready: %s" % state)
1968	else:
1969	raise service_error(service_error.req, "No such experiment")
1970
1971	def get_vis(self, req, fid):
1972	"""
1973	Return the stored visualization for this experiment
1974	"""
1975	rv = None
1976	state = None
1977
1978	req = req.get('VisRequestBody', None)
1979	if not req:
1980	raise service_error(service_error.req,
1981	"Bad request format (no VisRequestBody)")
1982	exp = req.get('experiment', None)
1983	if exp:
1984	if exp.has_key('fedid'):
1985	key = exp['fedid']
1986	keytype = "fedid"
1987	elif exp.has_key('localname'):
1988	key = exp['localname']
1989	keytype = "localname"
1990	else:
1991	raise service_error(service_error.req, "Unknown lookup type")
1992	else:
1993	raise service_error(service_error.req, "No request?")
1994
1995	self.check_experiment_access(fid, key)
1996
1997	self.state_lock.acquire()
1998	if self.state.has_key(key):
1999	if self.state[key].has_key('vis'):
2000	rv = { 'experiment' : {keytype: key },\
2001	'vis': self.state[key]['vis'],\
2002	}
2003	else:
2004	state = self.state[key]['experimentStatus']
2005	self.state_lock.release()
2006
2007	if rv: return rv
2008	else:
2009	if state:
2010	raise service_error(service_error.partial,
2011	"Not ready: %s" % state)
2012	else:
2013	raise service_error(service_error.req, "No such experiment")
2014
2015	def clean_info_response(self, rv):
2016	"""
2017	Remove the information in the experiment's state object that is not in
2018	the info response.
2019	"""
2020	# Remove the owner info (should always be there, but...)
2021	if rv.has_key('owner'): del rv['owner']
2022
2023	# Convert the log into the allocationLog parameter and remove the
2024	# log entry (with defensive programming)
2025	if rv.has_key('log'):
2026	rv['allocationLog'] = "".join(rv['log'])
2027	del rv['log']
2028	else:
2029	rv['allocationLog'] = ""
2030
2031	if rv['experimentStatus'] != 'active':
2032	if rv.has_key('federant'): del rv['federant']
2033	else:
2034	# remove the allocationID and uri info from each federant
2035	for f in rv.get('federant', []):
2036	if f.has_key('allocID'): del f['allocID']
2037	if f.has_key('uri'): del f['uri']
2038
2039	return rv
2040
2041	def get_info(self, req, fid):
2042	"""
2043	Return all the stored info about this experiment
2044	"""
2045	rv = None
2046
2047	req = req.get('InfoRequestBody', None)
2048	if not req:
2049	raise service_error(service_error.req,
2050	"Bad request format (no InfoRequestBody)")
2051	exp = req.get('experiment', None)
2052	if exp:
2053	if exp.has_key('fedid'):
2054	key = exp['fedid']
2055	keytype = "fedid"
2056	elif exp.has_key('localname'):
2057	key = exp['localname']
2058	keytype = "localname"
2059	else:
2060	raise service_error(service_error.req, "Unknown lookup type")
2061	else:
2062	raise service_error(service_error.req, "No request?")
2063
2064	self.check_experiment_access(fid, key)
2065
2066	# The state may be massaged by the service function that called
2067	# get_info (e.g., encoded for XMLRPC transport) so send a copy of the
2068	# state.
2069	self.state_lock.acquire()
2070	if self.state.has_key(key):
2071	rv = copy.deepcopy(self.state[key])
2072	self.state_lock.release()
2073
2074	if rv:
2075	return self.clean_info_response(rv)
2076	else:
2077	raise service_error(service_error.req, "No such experiment")
2078
2079	def get_multi_info(self, req, fid):
2080	"""
2081	Return all the stored info that this fedid can access
2082	"""
2083	rv = { 'info': [ ] }
2084
2085	self.state_lock.acquire()
2086	for key in [ k for k in self.state.keys() if isinstance(k, fedid)]:
2087	try:
2088	self.check_experiment_access(fid, key)
2089	except service_error, e:
2090	if e.code == service_error.access:
2091	continue
2092	else:
2093	self.state_lock.release()
2094	raise e
2095
2096	if self.state.has_key(key):
2097	e = copy.deepcopy(self.state[key])
2098	e = self.clean_info_response(e)
2099	rv['info'].append(e)
2100	self.state_lock.release()
2101	return rv
2102
2103	def terminate_experiment(self, req, fid):
2104	"""
2105	Swap this experiment out on the federants and delete the shared
2106	information
2107	"""
2108	tbparams = { }
2109	req = req.get('TerminateRequestBody', None)
2110	if not req:
2111	raise service_error(service_error.req,
2112	"Bad request format (no TerminateRequestBody)")
2113	force = req.get('force', False)
2114	exp = req.get('experiment', None)
2115	if exp:
2116	if exp.has_key('fedid'):
2117	key = exp['fedid']
2118	keytype = "fedid"
2119	elif exp.has_key('localname'):
2120	key = exp['localname']
2121	keytype = "localname"
2122	else:
2123	raise service_error(service_error.req, "Unknown lookup type")
2124	else:
2125	raise service_error(service_error.req, "No request?")
2126
2127	self.check_experiment_access(fid, key)
2128
2129	dealloc_list = [ ]
2130
2131
2132	# Create a logger that logs to the dealloc_list as well as to the main
2133	# log file.
2134	dealloc_log = logging.getLogger('fedd.experiment_control.%s' % key)
2135	h = logging.StreamHandler(self.list_log(dealloc_list))
2136	# XXX: there should be a global one of these rather than repeating the
2137	# code.
2138	h.setFormatter(logging.Formatter("%(asctime)s %(name)s %(message)s",
2139	'%d %b %y %H:%M:%S'))
2140	dealloc_log.addHandler(h)
2141
2142	self.state_lock.acquire()
2143	fed_exp = self.state.get(key, None)
2144
2145	if fed_exp:
2146	# This branch of the conditional holds the lock to generate a
2147	# consistent temporary tbparams variable to deallocate experiments.
2148	# It releases the lock to do the deallocations and reacquires it to
2149	# remove the experiment state when the termination is complete.
2150
2151	# First make sure that the experiment creation is complete.
2152	status = fed_exp.get('experimentStatus', None)
2153
2154	if status:
2155	if status in ('starting', 'terminating'):
2156	if not force:
2157	self.state_lock.release()
2158	raise service_error(service_error.partial,
2159	'Experiment still being created or destroyed')
2160	else:
2161	self.log.warning('Experiment in %s state ' % status + \
2162	'being terminated by force.')
2163	else:
2164	# No status??? trouble
2165	self.state_lock.release()
2166	raise service_error(service_error.internal,
2167	"Experiment has no status!?")
2168
2169	ids = []
2170	# experimentID is a list of dicts that are self-describing
2171	# identifiers. This finds all the fedids and localnames - the
2172	# keys of self.state - and puts them into ids.
2173	for id in fed_exp.get('experimentID', []):
2174	if id.has_key('fedid'): ids.append(id['fedid'])
2175	if id.has_key('localname'): ids.append(id['localname'])
2176
2177	# Collect the allocation/segment ids into a dict keyed by the fedid
2178	# of the allocation (or a monotonically increasing integer) that
2179	# contains a tuple of uri, aid (which is a dict...)
2180	for i, fed in enumerate(fed_exp.get('federant', [])):
2181	try:
2182	uri = fed['uri']
2183	aid = fed['allocID']
2184	k = fed['allocID'].get('fedid', i)
2185	except KeyError, e:
2186	continue
2187	tbparams[k] = (uri, aid)
2188	fed_exp['experimentStatus'] = 'terminating'
2189	if self.state_filename: self.write_state()
2190	self.state_lock.release()
2191
2192	# Stop everyone. NB, wait_for_all waits until a thread starts and
2193	# then completes, so we can't wait if nothing starts. So, no
2194	# tbparams, no start.
2195	if len(tbparams) > 0:
2196	thread_pool = self.thread_pool(self.nthreads)
2197	for k in tbparams.keys():
2198	# Create and start a thread to stop the segment
2199	thread_pool.wait_for_slot()
2200	uri, aid = tbparams[k]
2201	t = self.pooled_thread(\
2202	target=self.terminate_segment(log=dealloc_log,
2203	testbed=uri,
2204	cert_file=self.cert_file,
2205	cert_pwd=self.cert_pwd,
2206	trusted_certs=self.trusted_certs,
2207	caller=self.call_TerminateSegment),
2208	args=(uri, aid), name=k,
2209	pdata=thread_pool, trace_file=self.trace_file)
2210	t.start()
2211	# Wait for completions
2212	thread_pool.wait_for_all_done()
2213
2214	# release the allocations (failed experiments have done this
2215	# already, and starting experiments may be in odd states, so we
2216	# ignore errors releasing those allocations
2217	try:
2218	for k in tbparams.keys():
2219	# This releases access by uri
2220	uri, aid = tbparams[k]
2221	self.release_access(None, aid, uri=uri)
2222	except service_error, e:
2223	if status != 'failed' and not force:
2224	raise e
2225
2226	# Remove the terminated experiment
2227	self.state_lock.acquire()
2228	for id in ids:
2229	if self.state.has_key(id): del self.state[id]
2230
2231	if self.state_filename: self.write_state()
2232	self.state_lock.release()
2233
2234	# Delete any synch points associated with this experiment. All
2235	# synch points begin with the fedid of the experiment.
2236	fedid_keys = set(["fedid:%s" % f for f in ids \
2237	if isinstance(f, fedid)])
2238	for k in self.synch_store.all_keys():
2239	try:
2240	if len(k) > 45 and k[0:46] in fedid_keys:
2241	self.synch_store.del_value(k)
2242	except synch_store.BadDeletionError:
2243	pass
2244	self.write_store()
2245
2246	return {
2247	'experiment': exp ,
2248	'deallocationLog': "".join(dealloc_list),
2249	}
2250	else:
2251	# Don't forget to release the lock
2252	self.state_lock.release()
2253	raise service_error(service_error.req, "No saved state")
2254
2255
2256	def GetValue(self, req, fid):
2257	"""
2258	Get a value from the synchronized store
2259	"""
2260	req = req.get('GetValueRequestBody', None)
2261	if not req:
2262	raise service_error(service_error.req,
2263	"Bad request format (no GetValueRequestBody)")
2264
2265	name = req['name']
2266	wait = req['wait']
2267	rv = { 'name': name }
2268
2269	if self.auth.check_attribute(fid, name):
2270	self.log.debug("[GetValue] asking for %s " % name)
2271	try:
2272	v = self.synch_store.get_value(name, wait)
2273	except synch_store.RevokedKeyError:
2274	# No more synch on this key
2275	raise service_error(service_error.federant,
2276	"Synch key %s revoked" % name)
2277	if v is not None:
2278	rv['value'] = v
2279	self.log.debug("[GetValue] got %s from %s" % (v, name))
2280	return rv
2281	else:
2282	raise service_error(service_error.access, "Access Denied")
2283
2284
2285	def SetValue(self, req, fid):
2286	"""
2287	Set a value in the synchronized store
2288	"""
2289	req = req.get('SetValueRequestBody', None)
2290	if not req:
2291	raise service_error(service_error.req,
2292	"Bad request format (no SetValueRequestBody)")
2293
2294	name = req['name']
2295	v = req['value']
2296
2297	if self.auth.check_attribute(fid, name):
2298	try:
2299	self.synch_store.set_value(name, v)
2300	self.write_store()
2301	self.log.debug("[SetValue] set %s to %s" % (name, v))
2302	except synch_store.CollisionError:
2303	# Translate into a service_error
2304	raise service_error(service_error.req,
2305	"Value already set: %s" %name)
2306	except synch_store.RevokedKeyError:
2307	# No more synch on this key
2308	raise service_error(service_error.federant,
2309	"Synch key %s revoked" % name)
2310	return { 'name': name, 'value': v }
2311	else:
2312	raise service_error(service_error.access, "Access Denied")

Note: See TracBrowser for help on using the repository browser.

Download in other formats: