Context Navigation

source: fedd/fedd_create_experiment.py @ 03e0290

axis_examplecompt_changesinfo-opsversion-1.30version-2.00version-3.01version-3.02

Last change on this file since 03e0290 was 987aaa1, checked in by Ted Faber <faber@…>, 16 years ago
get topo and vis data, persistent state
Property mode set to `100644`
File size: 40.0 KB

Line
1	#!/usr/local/bin/python
2
3	import os,sys
4
5	from ZSI import *
6	from M2Crypto import SSL
7	from M2Crypto.SSL.SSLServer import SSLServer
8	import M2Crypto.httpslib
9
10	import xml.parsers.expat
11
12	import re
13	import random
14	import string
15	import subprocess
16	import tempfile
17	import copy
18
19	import traceback
20
21	from threading import *
22
23	from subprocess import *
24
25	from fedd_services import *
26	from fedd_internal_services import *
27	from fedd_util import *
28	import parse_detail
29	from service_error import *
30
31	class fedd_create_experiment_local:
32	scripts = ["fed_bootstrap", "federate.sh", "smbmount.FreeBSD.pl",
33	"smbmount.Linux.pl", "make_hosts", "fed-tun.pl", "fed-tun.ucb.pl",
34	"fed_evrepeater", "rc.accounts.patch"]
35
36	def __init__(self,
37	cert_file=None,
38	cert_pwd=None,
39	exp_stem="faber-splitter",
40	debug=False,
41	muxmax=2,
42	nthreads=2,
43	randomize_experiments=False,
44	scp_exec="/usr/bin/scp",
45	scripts_dir="./",
46	splitter=None,
47	ssh_exec="/usr/bin/ssh",
48	ssh_identity_file=None,
49	ssh_keygen="/usr/bin/ssh-keygen",
50	ssh_pubkey_file=None,
51	ssh_type="rsa",
52	tbmap=None,
53	tclsh="/usr/local/bin/otclsh",
54	tcl_splitter="/usr/testbed/lib/ns2ir/parse.tcl",
55	trace_file=None,
56	trusted_certs=None,
57	):
58	self.scripts = fedd_create_experiment_local.scripts
59	self.thread_with_rv = fedd_create_experiment_local.pooled_thread
60	self.thread_pool = fedd_create_experiment_local.thread_pool
61
62	self.cert_file = cert_file
63	self.cert_pwd = cert_pwd
64	self.exp_stem = exp_stem
65	self.debug = debug
66	self.muxmax = muxmax
67	self.nthreads = nthreads
68	self.randomize_experiments = randomize_experiments
69	self.scp_exec = scp_exec
70	self.scripts_dir = scripts_dir
71	self.splitter = splitter
72	self.ssh_exec=ssh_exec
73	self.ssh_keygen = ssh_keygen
74	self.ssh_identity_file = ssh_identity_file
75	self.ssh_type = ssh_type
76	self.state = { }
77	self.tclsh = tclsh
78	self.tcl_splitter = tcl_splitter
79	self.tbmap = tbmap
80	self.trace_file = trace_file
81	self.trusted_certs=trusted_certs
82
83	self.def_expstart = \
84	"sudo -H /bin/sh FEDDIR/fed_bootstrap >& /tmp/federate";
85	self.def_mexpstart = "sudo -H FEDDIR/make_hosts FEDDIR/hosts";
86	self.def_gwstart = \
87	"sudo -H FEDDIR/fed-tun.pl -f GWCONF>& /tmp/bridge.log";
88	self.def_mgwstart = \
89	"sudo -H FEDDIR/fed-tun.pl -f GWCONF >& /tmp/bridge.log";
90	self.def_gwimage = "FBSD61-TUNNEL2";
91	self.def_gwtype = "pc";
92
93
94	if ssh_pubkey_file:
95	try:
96	f = open(ssh_pubkey_file, 'r')
97	self.ssh_pubkey = f.read()
98	f.close()
99	except IOError:
100	raise service_error(service_error.internal,
101	"Cannot read sshpubkey")
102
103	# Confirm federation scripts in the right place
104	for s in self.scripts:
105	if not os.path.exists(self.scripts_dir + "/" + s):
106	raise service_error(service_error.server_config,
107	"%s/%s not in local script dir" % (self.scripts_dir, s))
108	class thread_pool:
109	def __init__(self):
110	self.changed = Condition()
111	self.started = 0
112	self.terminated = 0
113
114	def acquire(self):
115	self.changed.acquire()
116
117	def release(self):
118	self.changed.release()
119
120	def wait(self, timeout = None):
121	self.changed.wait(timeout)
122
123	def start(self):
124	self.changed.acquire()
125	self.started += 1
126	self.changed.notifyAll()
127	self.changed.release()
128
129	def terminate(self):
130	self.changed.acquire()
131	self.terminated += 1
132	self.changed.notifyAll()
133	self.changed.release()
134
135	def clear(self):
136	self.changed.acquire()
137	self.started = 0
138	self.terminated =0
139	self.changed.notifyAll()
140	self.changed.release()
141
142
143
144	class pooled_thread(Thread):
145	def __init__(self, group=None, target=None, name=None, args=(),
146	kwargs={}, pdata=None, trace_file=None):
147	Thread.__init__(self, group, target, name, args, kwargs)
148	self.rv = None
149	self.exception = None
150	self.target=target
151	self.args = args
152	self.kwargs = kwargs
153	self.pdata = pdata
154	self.trace_file = trace_file
155
156	def run(self):
157	if self.pdata:
158	self.pdata.start()
159
160	if self.target:
161	try:
162	self.rv = self.target(self.args, *self.kwargs)
163	except service_error, s:
164	self.exception = s
165	if self.trace_file:
166	print >>self.trace_file, "Thread exception: %s %s" % \
167	(s.code_string(), s.desc)
168
169	except:
170	self.exception = sys.exc_info()[1]
171	if self.trace_file:
172	print >>self.trace_file, \
173	"Unexpected thread exception: %s" % \
174	self.exception
175	print >>self.trace_file, "Trace: %s" % \
176	traceback.format_exc()
177	if self.pdata:
178	self.pdata.terminate()
179
180	def copy_file(self, src, dest, size=1024):
181	"""
182	Exceedingly simple file copy.
183	"""
184	s = open(src,'r')
185	d = open(dest, 'w')
186
187	buf = "x"
188	while buf != "":
189	buf = s.read(size)
190	d.write(buf)
191	s.close()
192	d.close()
193
194	def scp_file(self, file, user, host, dest=""):
195	"""
196	scp a file to the remote host.
197	"""
198
199	scp_cmd = [self.scp_exec, file, "%s@%s:%s" % (user, host, dest)]
200
201	trace = self.trace_file
202	if not trace:
203	try:
204	trace = open("/dev/null", "w")
205	except IOError:
206	raise service_error(service_error.internal,
207	"Cannot open /dev/null??");
208
209	if not self.debug:
210	rv = call(scp_cmd, stdout=trace, stderr=trace)
211	else:
212	if self.trace_file:
213	print >>self.trace_file, "debug [scp_file]: %s" % \
214	" ".join(scp_cmd)
215	rv = 0
216
217	return rv == 0
218
219	def ssh_cmd(self, user, host, cmd, wname=None):
220	sh_str = "%s %s@%s %s" % (self.ssh_exec, user, host, cmd)
221
222	trace = self.trace_file
223	if not trace:
224	try:
225	trace = open("/dev/null", "w")
226	except IOError:
227	raise service_error(service_error.internal,
228	"Cannot open /dev/null??");
229
230	if not self.debug:
231	sub = Popen(sh_str, shell=True, stdout=trace, stderr=trace)
232	return sub.wait() == 0
233	else:
234	if self.trace_file:
235	print >>self.trace_file,"debug [ssh_cmd]: %s" % sh_str
236	return True
237
238	def ship_scripts(self, host, user, dest_dir):
239	if self.ssh_cmd(user, host, "mkdir -p %s" % dest_dir):
240	for s in self.scripts:
241	if not self.scp_file("%s/%s" % (self.scripts_dir, s),
242	user, host, dest_dir):
243	return False
244	return True
245	else:
246	return False
247
248	def ship_configs(self, host, user, src_dir, dest_dir):
249	if not self.ssh_cmd(user, host, "mkdir -p %s" % dest_dir):
250	return False
251	if not self.ssh_cmd(user, host, "chmod 770 %s" % dest_dir):
252	return False
253
254	for f in os.listdir(src_dir):
255	if os.path.isdir(f):
256	if not self.ship_configs(host, user, "%s/%s" % (src_dir, f),
257	"%s/%s" % (dest_dir, f)):
258	return False
259	else:
260	if not self.scp_file("%s/%s" % (src_dir, f),
261	user, host, dest_dir):
262	return False
263	return True
264
265	def start_segment(self, tb, eid, tbparams, tmpdir, timeout=0):
266	host = "%s%s" % (tbparams[tb]['host'], tbparams[tb]['domain'])
267	user = tbparams[tb]['user']
268	pid = tbparams[tb]['project']
269	# XXX
270	base_confs = ( "hosts",)
271	tclfile = "%s.%s.tcl" % (eid, tb)
272	expinfo_exec = "/usr/testbed/bin/expinfo"
273	proj_dir = "/proj/%s/exp/%s/tmp" % (pid, eid)
274	tarfiles_dir = "/proj/%s/tarfiles/%s" % (pid, eid)
275	rpms_dir = "/proj/%s/rpms/%s" % (pid, eid)
276	state_re = re.compile("State:\s+(\w+)")
277	no_exp_re = re.compile("^No\s+such\s+experiment")
278	state = None
279	cmd = [self.ssh_exec, "%s@%s" % (user, host), expinfo_exec, pid, eid]
280
281
282	if self.trace_file:
283	print >>self.trace_file, "status request: %s" % " ".join(cmd)
284
285	if not self.trace_file:
286	try:
287	st_file = open("/dev/null", "w")
288	except IOError:
289	raise service_error(service_error.internal,
290	"Cannot open /dev/null!?")
291	else:
292	st_file = self.trace_file
293
294	status = Popen(cmd, stdout=PIPE, stderr=st_file)
295	for line in status.stdout:
296	m = state_re.match(line)
297	if m: state = m.group(1)
298	else:
299	m = no_exp_re.match(line)
300	if m: state = "none"
301	rv = status.wait()
302	# If the experiment is not present the subcommand returns a non-zero
303	# return value. If we successfully parsed a "none" outcome, ignore the
304	# return code.
305	if rv != 0 and state != "none":
306	raise service_error(service_error.internal,
307	"Cannot get status of segment %s:%s/%s" % (tb, pid, eid))
308	if self.trace_file:
309	print >>self.trace_file, "%s: %s" % (tb, state)
310	print >>self.trace_file, "transferring experiment to %s" % tb
311
312	if not self.scp_file("%s/%s/%s" % (tmpdir, tb, tclfile), user, host):
313	return False
314	# Clear and create the tarfiles and rpm directories
315	for d in (tarfiles_dir, rpms_dir):
316	if not self.ssh_cmd(user, host,
317	"/bin/sh -c \"'/bin/rm -rf %s/*'\"" % d):
318	return False
319	if not self.ssh_cmd(user, host, "mkdir -p %s" % d,
320	"create tarfiles"):
321	return False
322
323	if state == 'active':
324	# Remote experiment is active. Modify it.
325	for f in base_confs:
326	if not self.scp_file("%s/%s" % (tmpdir, f), user, host,
327	"%s/%s" % (proj_dir, f)):
328	return False
329	if not self.ship_scripts(host, user, proj_dir):
330	return False
331	if not self.ship_configs(host, user, "%s/%s" % (tmpdir, tb),
332	proj_dir):
333	return False
334	if os.path.isdir("%s/tarfiles" % tmpdir):
335	if not self.ship_configs(host, user,
336	"%s/tarfiles" % tmpdir, tarfiles_dir):
337	return False
338	if os.path.isdir("%s/rpms" % tmpdir):
339	if not self.ship_configs(host, user,
340	"%s/rpms" % tmpdir, tarfiles_dir):
341	return False
342	if self.trace_file:
343	print >>self.trace_file, "Modifying %s on %s" % (eid, tb)
344	if not self.ssh_cmd(user, host,
345	"/usr/testbed/bin/modexp -r -s -w %s %s %s" % \
346	(pid, eid, tclfile), "modexp"):
347	return False
348	return True
349	elif state == "swapped":
350	# Remote experiment swapped out. Modify it and swap it in.
351	for f in base_confs:
352	if not self.scp_file("%s/%s" % (tmpdir, f), user, host,
353	"%s/%s" % (proj_dir, f)):
354	return False
355	if not self.ship_scripts(host, user, proj_dir):
356	return False
357	if not self.ship_configs(host, user, "%s/%s" % (tmpdir, tb),
358	proj_dir):
359	return False
360	if os.path.isdir("%s/tarfiles" % tmpdir):
361	if not self.ship_configs(host, user,
362	"%s/tarfiles" % tmpdir, tarfiles_dir):
363	return False
364	if os.path.isdir("%s/rpms" % tmpdir):
365	if not self.ship_configs(host, user,
366	"%s/rpms" % tmpdir, tarfiles_dir):
367	return False
368	if self.trace_file:
369	print >>self.trace_file, "Modifying %s on %s" % (eid, tb)
370	if not self.ssh_cmd(user, host,
371	"/usr/testbed/bin/modexp -w %s %s %s" % (pid, eid, tclfile),
372	"modexp"):
373	return False
374	if self.trace_file:
375	print >>self.trace_file, "Swapping %s in on %s" % (eid, tb)
376	if not self.ssh_cmd(user, host,
377	"/usr/testbed/bin/swapexp -w %s %s in" % (pid, eid),
378	"swapexp"):
379	return False
380	return True
381	elif state == "none":
382	# No remote experiment. Create one. We do this in 2 steps so we
383	# can put the configuration files and scripts into the new
384	# experiment directories.
385
386	# Tarfiles must be present for creation to work
387	if os.path.isdir("%s/tarfiles" % tmpdir):
388	if not self.ship_configs(host, user,
389	"%s/tarfiles" % tmpdir, tarfiles_dir):
390	return False
391	if os.path.isdir("%s/rpms" % tmpdir):
392	if not self.ship_configs(host, user,
393	"%s/rpms" % tmpdir, tarfiles_dir):
394	return False
395	if self.trace_file:
396	print >>self.trace_file, "Creating %s on %s" % (eid, tb)
397	if not self.ssh_cmd(user, host,
398	"/usr/testbed/bin/startexp -i -f -w -p %s -e %s %s" % \
399	(pid, eid, tclfile), "startexp"):
400	return False
401	# After startexp the per-experiment directories exist
402	for f in base_confs:
403	if not self.scp_file("%s/%s" % (tmpdir, f), user, host,
404	"%s/%s" % (proj_dir, f)):
405	return False
406	if not self.ship_scripts(host, user, proj_dir):
407	return False
408	if not self.ship_configs(host, user, "%s/%s" % (tmpdir, tb),
409	proj_dir):
410	return False
411	if self.trace_file:
412	print >>self.trace_file, "Swapping %s in on %s" % (eid, tb)
413	if not self.ssh_cmd(user, host,
414	"/usr/testbed/bin/swapexp -w %s %s in" % (pid, eid),
415	"swapexp"):
416	return False
417	return True
418	else:
419	if self.trace_file:
420	print >>self.trace_file, "unknown state %s" % state
421	return False
422
423	def stop_segment(self, tb, eid, tbparams):
424	user = tbparams[tb]['user']
425	host = "%s%s" % (tbparams[tb]['host'], tbparams[tb]['domain'])
426	pid = tbparams[tb]['project']
427
428	if self.trace_file:
429	print >>self.trace_file, "Stopping %s on %s" % (eid, tb)
430	return self.ssh_cmd(user, host,
431	"/usr/testbed/bin/swapexp -w %s %s out" % (pid, eid))
432
433
434	def generate_ssh_keys(self, dest, type="rsa" ):
435	"""
436	Generate a set of keys for the gateways to use to talk.
437
438	Keys are of type type and are stored in the required dest file.
439	"""
440	valid_types = ("rsa", "dsa")
441	t = type.lower();
442	if t not in valid_types: raise ValueError
443
444	trace = self.trace_file
445	if not trace:
446	try:
447	trace = open("/dev/null", "w")
448	except IOError:
449	raise service_error(service_error.internal,
450	"Cannot open /dev/null??");
451
452	# May raise CalledProcessError
453	rv = call([self.ssh_keygen, '-t', t, '-N', '', '-f', dest],
454	stdout=trace, stderr=trace)
455	if rv != 0:
456	raise service_error(service_error.internal,
457	"Cannot generate nonce ssh keys. %s return code %d" \
458	% (self.ssh_keygen, rv))
459
460	def gentopo(self, str):
461	class topo_parse:
462	def __init__(self):
463	self.str_subelements = ('vname', 'vnode', 'ips', 'ip', 'member')
464	self.int_subelements = ( 'bandwidth',)
465	self.float_subelements = ( 'delay',)
466	self.nodes = [ ]
467	self.lans = [ ]
468	self.element = { }
469	self.topo = { \
470	'node': self.nodes,\
471	'lan' : self.lans,\
472	}
473	self.chars = ""
474
475	def end_element(self, name):
476	if name == 'node':
477	self.nodes.append(self.element)
478	self.element = { }
479	elif name == 'lan':
480	self.lans.append(self.element)
481	self.element = { }
482	elif name in self.str_subelements:
483	self.element[name] = self.chars
484	self.chars = ""
485	elif name in self.int_subelements:
486	self.element[name] = int(self.chars)
487	self.chars = ""
488	elif name in self.float_subelements:
489	self.element[name] = float(self.chars)
490	self.chars = ""
491
492	def found_chars(self, data):
493	self.chars += data.rstrip()
494
495
496	tp = topo_parse();
497	parser = xml.parsers.expat.ParserCreate()
498	parser.EndElementHandler = tp.end_element
499	parser.CharacterDataHandler = tp.found_chars
500
501	parser.Parse(str)
502
503	return tp.topo
504
505
506	def genviz(self, topo):
507	"""
508	Generate the visualization the virtual topology
509	"""
510
511	neato = "/usr/local/bin/neato"
512	# These are used to parse neato output and to create the visualization
513	# file.
514	vis_re = re.compile('^\s"?([\w\-]+)"?\s+\[.pos="(\d+),(\d+)"')
515	vis_fmt = "<node><name>%s</name><x>%s</x><y>%s</y><type>" + \
516	"%s</type></node>"
517
518	try:
519	# Node names
520	nodes = [ n['vname'] for n in topo['node'] ]
521	topo_lans = topo['lan']
522	except KeyError:
523	raise service_error(service_error.internal, "Bad topology")
524
525	lans = { }
526	links = { }
527
528	# Walk through the virtual topology, organizing the connections into
529	# 2-node connections (links) and more-than-2-node connections (lans).
530	# When a lan is created, it's added to the list of nodes (there's a
531	# node in the visualization for the lan).
532	for l in topo_lans:
533	if links.has_key(l['vname']):
534	if len(links[l['vname']]) < 2:
535	links[l['vname']].append(l['vnode'])
536	else:
537	nodes.append(l['vname'])
538	lans[l['vname']] = links[l['vname']]
539	del links[l['vname']]
540	lans[l['vname']].append(l['vnode'])
541	elif lans.has_key(l['vname']):
542	lans[l['vname']].append(l['vnode'])
543	else:
544	links[l['vname']] = [ l['vnode'] ]
545
546
547	# Open up a temporary file for dot to turn into a visualization
548	try:
549	df, dotname = tempfile.mkstemp()
550	dotfile = os.fdopen(df, 'w')
551	except IOError:
552	raise service_error(service_error.internal,
553	"Failed to open file in genviz")
554
555	# Generate a dot/neato input file from the links, nodes and lans
556	try:
557	print >>dotfile, "graph G {"
558	for n in nodes:
559	print >>dotfile, '\t"%s"' % n
560	for l in links.keys():
561	print >>dotfile, '\t"%s" -- "%s"' % tuple(links[l])
562	for l in lans.keys():
563	for n in lans[l]:
564	print >>dotfile, '\t "%s" -- "%s"' % (n,l)
565	print >>dotfile, "}"
566	dotfile.close()
567	except TypeError:
568	raise service_error(service_error.internal,
569	"Single endpoint link in vtopo")
570	except IOError:
571	raise service_error(service_error.internal, "Cannot write dot file")
572
573	# Use dot to create a visualization
574	dot = Popen([neato, '-Gstart=rand', '-Gepsilon=0.005', '-Gmaxiter=2000',
575	'-Gpack=true', dotname], stdout=PIPE)
576
577	# Translate dot to vis format
578	vis_nodes = [ ]
579	vis = { 'node': vis_nodes }
580	for line in dot.stdout:
581	m = vis_re.match(line)
582	if m:
583	vn = m.group(1)
584	vis_node = {'name': vn, \
585	'x': float(m.group(2)),\
586	'y' : float(m.group(3)),\
587	}
588	if vn in links.keys() or vn in lans.keys():
589	vis_node['type'] = 'lan'
590	else:
591	vis_node['type'] = 'node'
592	vis_nodes.append(vis_node)
593	rv = dot.wait()
594
595	os.remove(dotname)
596	if rv == 0 : return vis
597	else: return None
598
599
600	def get_access(self, tb, nodes, user, tbparam):
601	"""
602	Get access to testbed through fedd and set the parameters for that tb
603	"""
604
605	translate_attr = {
606	'slavenodestartcmd': 'expstart',
607	'slaveconnectorstartcmd': 'gwstart',
608	'masternodestartcmd': 'mexpstart',
609	'masterconnectorstartcmd': 'mgwstart',
610	'connectorimage': 'gwimage',
611	'connectortype': 'gwtype',
612	'tunnelcfg': 'tun',
613	'smbshare': 'smbshare',
614	}
615
616	# XXX multi-level access
617	uri = self.tbmap.get(tb, None)
618	if not uri:
619	raise service_error(serice_error.server_config,
620	"Unknown testbed: %s" % tb)
621
622	# The basic request
623	req = {\
624	'destinationTestbed' : { 'uri' : uri },
625	'user': user,
626	'allocID' : { 'username': 'test' },
627	'access' : [ { 'sshPubkey' : self.ssh_pubkey } ]
628	}
629
630	# node resources if any
631	if nodes != None and len(nodes) > 0:
632	rnodes = [ ]
633	for n in nodes:
634	rn = { }
635	image, hw, count = n.split(":")
636	if image: rn['image'] = [ image ]
637	if hw: rn['hardware'] = [ hw ]
638	if count: rn['count'] = int(count)
639	rnodes.append(rn)
640	req['resources']= { }
641	req['resources']['node'] = rnodes
642
643	# No retry loop here. Proxy servers must correctly authenticate
644	# themselves without help
645	try:
646	ctx = fedd_ssl_context(self.cert_file,
647	self.trusted_certs, password=self.cert_pwd)
648	except SSL.SSLError:
649	raise service_error(service_error.server_config,
650	"Server certificates misconfigured")
651
652	loc = feddServiceLocator();
653	port = loc.getfeddPortType(uri,
654	transport=M2Crypto.httpslib.HTTPSConnection,
655	transdict={ 'ssl_context' : ctx })
656
657	# Reconstruct the full request message
658	msg = RequestAccessRequestMessage()
659	msg.set_element_RequestAccessRequestBody(
660	pack_soap(msg, "RequestAccessRequestBody", req))
661	try:
662	resp = port.RequestAccess(msg)
663	except ZSI.ParseException, e:
664	raise service_error(service_error.req,
665	"Bad format message (XMLRPC??): %s" %
666	str(e))
667	r = unpack_soap(resp)
668
669	if r.has_key('RequestAccessResponseBody'):
670	r = r['RequestAccessResponseBody']
671	else:
672	raise service_error(service_error.proxy,
673	"Bad proxy response")
674
675
676	e = r['emulab']
677	p = e['project']
678	tbparam[tb] = {
679	"boss": e['boss'],
680	"host": e['ops'],
681	"domain": e['domain'],
682	"fs": e['fileServer'],
683	"eventserver": e['eventServer'],
684	"project": unpack_id(p['name']),
685	"emulab" : e
686	}
687
688	for u in p['user']:
689	tbparam[tb]['user'] = unpack_id(u['userID'])
690
691	for a in e['fedAttr']:
692	if a['attribute']:
693	key = translate_attr.get(a['attribute'].lower(), None)
694	if key:
695	tbparam[tb][key]= a['value']
696
697	class current_testbed:
698	def __init__(self, eid, tmpdir):
699	self.begin_testbed = re.compile("^#\s+Begin\s+Testbed\s+\((\w+)\)")
700	self.end_testbed = re.compile("^#\s+End\s+Testbed\s+\((\w+)\)")
701	self.current_testbed = None
702	self.testbed_file = None
703
704	self.def_expstart = \
705	"sudo -H /bin/sh FEDDIR/fed_bootstrap >& /tmp/federate";
706	self.def_mexpstart = "sudo -H FEDDIR/make_hosts FEDDIR/hosts";
707	self.def_gwstart = \
708	"sudo -H FEDDIR/fed-tun.pl -f GWCONF>& /tmp/bridge.log";
709	self.def_mgwstart = \
710	"sudo -H FEDDIR/fed-tun.pl -f GWCONF >& /tmp/bridge.log";
711	self.def_gwimage = "FBSD61-TUNNEL2";
712	self.def_gwtype = "pc";
713
714	self.eid = eid
715	self.tmpdir = tmpdir
716
717	def __call__(self, line, master, allocated, tbparams):
718	# Capture testbed topology descriptions
719	if self.current_testbed == None:
720	m = self.begin_testbed.match(line)
721	if m != None:
722	self.current_testbed = m.group(1)
723	if self.current_testbed == None:
724	raise service_error(service_error.req,
725	"Bad request format (unnamed testbed)")
726	allocated[self.current_testbed] = \
727	allocated.get(self.current_testbed,0) + 1
728	tb_dir = "%s/%s" % (self.tmpdir, self.current_testbed)
729	if not os.path.exists(tb_dir):
730	try:
731	os.mkdir(tb_dir)
732	except IOError:
733	raise service_error(service_error.internal,
734	"Cannot create %s" % tb_dir)
735	try:
736	self.testbed_file = open("%s/%s.%s.tcl" %
737	(tb_dir, self.eid, self.current_testbed), 'w')
738	except IOError:
739	self.testbed_file = None
740	return True
741	else: return False
742	else:
743	m = self.end_testbed.match(line)
744	if m != None:
745	if m.group(1) != self.current_testbed:
746	raise service_error(service_error.internal,
747	"Mismatched testbed markers!?")
748	if self.testbed_file != None:
749	self.testbed_file.close()
750	self.testbed_file = None
751	self.current_testbed = None
752	elif self.testbed_file:
753	# Substitute variables and put the line into the local
754	# testbed file.
755	gwtype = tbparams[self.current_testbed].get('gwtype',
756	self.def_gwtype)
757	gwimage = tbparams[self.current_testbed].get('gwimage',
758	self.def_gwimage)
759	mgwstart = tbparams[self.current_testbed].get('mgwstart',
760	self.def_mgwstart)
761	mexpstart = tbparams[self.current_testbed].get('mexpstart',
762	self.def_mexpstart)
763	gwstart = tbparams[self.current_testbed].get('gwstart',
764	self.def_gwstart)
765	expstart = tbparams[self.current_testbed].get('expstart',
766	self.def_expstart)
767	project = tbparams[self.current_testbed].get('project')
768	line = re.sub("GWTYPE", gwtype, line)
769	line = re.sub("GWIMAGE", gwimage, line)
770	if self.current_testbed == master:
771	line = re.sub("GWSTART", mgwstart, line)
772	line = re.sub("EXPSTART", mexpstart, line)
773	else:
774	line = re.sub("GWSTART", gwstart, line)
775	line = re.sub("EXPSTART", expstart, line)
776	# XXX: does `` embed without doing enything else?
777	line = re.sub("GWCONF", "FEDDIR`hostname`.gw.conf", line)
778	line = re.sub("PROJDIR", "/proj/%s/" % project, line)
779	line = re.sub("EID", self.eid, line)
780	line = re.sub("FEDDIR", "/proj/%s/exp/%s/tmp/" % \
781	(project, self.eid), line)
782	print >>self.testbed_file, line
783	return True
784
785	class allbeds:
786	def __init__(self, get_access):
787	self.begin_allbeds = re.compile("^#\s+Begin\s+Allbeds")
788	self.end_allbeds = re.compile("^#\s+End\s+Allbeds")
789	self.in_allbeds = False
790	self.get_access = get_access
791
792	def __call__(self, line, user, tbparams):
793	# Testbed access parameters
794	if not self.in_allbeds:
795	if self.begin_allbeds.match(line):
796	self.in_allbeds = True
797	return True
798	else:
799	return False
800	else:
801	if self.end_allbeds.match(line):
802	self.in_allbeds = False
803	else:
804	nodes = line.split('\|')
805	tb = nodes.pop(0)
806	self.get_access(tb, nodes, user, tbparams)
807	return True
808
809	class gateways:
810	def __init__(self, eid, master, tmpdir, gw_pubkey,
811	gw_secretkey, copy_file):
812	self.begin_gateways = \
813	re.compile("^#\s+Begin\s+gateways\s+\((\w+)\)")
814	self.end_gateways = re.compile("^#\s+End\s+gateways\s+\((\w+)\)")
815	self.current_gateways = None
816	self.control_gateway = None
817	self.active_end = { }
818
819	self.eid = eid
820	self.master = master
821	self.tmpdir = tmpdir
822	self.gw_pubkey_base = gw_pubkey
823	self.gw_secretkey_base = gw_secretkey
824
825	self.copy_file = copy_file
826
827
828	def gateway_conf_file(self, gw, master, eid, pubkey, privkey,
829	active_end, tbparams, dtb, myname, desthost, type):
830	"""
831	Produce a gateway configuration file from a gateways line.
832	"""
833
834	sproject = tbparams[gw].get('project', 'project')
835	dproject = tbparams[dtb].get('project', 'project')
836	sdomain = ".%s.%s%s" % (eid, sproject,
837	tbparams[gw].get('domain', ".example.com"))
838	ddomain = ".%s.%s%s" % (eid, dproject,
839	tbparams[dtb].get('domain', ".example.com"))
840	boss = tbparams[master].get('boss', "boss")
841	fs = tbparams[master].get('fs', "fs")
842	event_server = "%s%s" % \
843	(tbparams[master].get('eventserver', "event_server"),
844	tbparams[master].get('domain', "example.com"))
845	remote_event_server = "%s%s" % \
846	(tbparams[dtb].get('eventserver', "event_server"),
847	tbparams[dtb].get('domain', "example.com"))
848
849	remote_script_dir = "/proj/%s/exp/%s/tmp" % ( dproject, eid)
850	local_script_dir = "/proj/%s/exp/%s/tmp" % ( sproject, eid)
851	tunnel_cfg = tbparams[gw].get("tun", "false")
852
853	conf_file = "%s%s.gw.conf" % (myname, sdomain)
854	remote_conf_file = "%s%s.gw.conf" % (desthost, ddomain)
855
856	# translate to lower case so the `hostname` hack for specifying
857	# configuration files works.
858	conf_file = conf_file.lower();
859	remote_conf_file = remote_conf_file.lower();
860
861	if dtb == master:
862	active = "false"
863	elif gw == master:
864	active = "true"
865	elif active_end.has_key['%s-%s' % (dtb, gw)]:
866	active = "false"
867	else:
868	active_end['%s-%s' % (gw, dtb)] = 1
869	active = "true"
870
871	gwconfig = open("%s/%s/%s" % (self.tmpdir, gw, conf_file), "w")
872	print >>gwconfig, "Active: %s" % active
873	print >>gwconfig, "TunnelCfg: %s" % tunnel_cfg
874	print >>gwconfig, "BossName: %s" % boss
875	print >>gwconfig, "FsName: %s" % fs
876	print >>gwconfig, "EventServerName: %s" % event_server
877	print >>gwconfig, "RemoteEventServerName: %s" % remote_event_server
878	print >>gwconfig, "Type: %s" % type
879	print >>gwconfig, "RemoteScriptDir: %s" % remote_script_dir
880	print >>gwconfig, "EventRepeater: %s/fed_evrepeater" % \
881	local_script_dir
882	print >>gwconfig, "RemoteExperiment: %s/%s" % (dproject, eid)
883	print >>gwconfig, "LocalExperiment: %s/%s" % (sproject, eid)
884	print >>gwconfig, "RemoteConfigFile: %s/%s" % \
885	(remote_script_dir, remote_conf_file)
886	print >>gwconfig, "Peer: %s%s" % (desthost, ddomain)
887	print >>gwconfig, "Pubkeys: %s/%s" % (local_script_dir, pubkey)
888	print >>gwconfig, "Privkeys: %s/%s" % (local_script_dir, privkey)
889	gwconfig.close()
890
891	return active == "true"
892
893	def __call__(self, line, allocated, tbparams):
894	# Process gateways
895	if not self.current_gateways:
896	m = self.begin_gateways.match(line)
897	if m:
898	self.current_gateways = m.group(1)
899	if allocated.has_key(self.current_gateways):
900	# This test should always succeed
901	tb_dir = "%s/%s" % (self.tmpdir, self.current_gateways)
902	if not os.path.exists(tb_dir):
903	try:
904	os.mkdir(tb_dir)
905	except IOError:
906	raise service_error(service_error.internal,
907	"Cannot create %s" % tb_dir)
908	else:
909	# XXX
910	print >>sys.stderr, \
911	"Ignoring gateways for unknown testbed %s" \
912	% self.current_gateways
913	self.current_gateways = None
914	return True
915	else:
916	return False
917	else:
918	m = self.end_gateways.match(line)
919	if m :
920	if m.group(1) != self.current_gateways:
921	raise service_error(service_error.internal,
922	"Mismatched gateway markers!?")
923	if self.control_gateway:
924	try:
925	cc = open("%s/%s/client.conf" %
926	(self.tmpdir, self.current_gateways), 'w')
927	print >>cc, "ControlGateway: %s" % \
928	self.control_gateway
929	if tbparams[self.master].has_key('smbshare'):
930	print >>cc, "SMBSHare: %s" % \
931	tbparams[self.master]['smbshare']
932	print >>cc, "ProjectUser: %s" % \
933	tbparams[self.master]['user']
934	print >>cc, "ProjectName: %s" % \
935	tbparams[self.master]['project']
936	cc.close()
937	except IOError:
938	raise service_error(service_error.internal,
939	"Error creating client config")
940	else:
941	if self.trace_file:
942	print >>sys.stderr, "No control gateway for %s" %\
943	self.current_gateways
944	self.current_gateways = None
945	else:
946	dtb, myname, desthost, type = line.split(" ")
947
948	if type == "control" or type == "both":
949	self.control_gateway = "%s.%s.%s%s" % (myname,
950	self.eid,
951	tbparams[self.current_gateways]['project'],
952	tbparams[self.current_gateways]['domain'])
953	try:
954	active = self.gateway_conf_file(self.current_gateways,
955	self.master, self.eid, self.gw_pubkey_base,
956	self.gw_secretkey_base,
957	self.active_end, tbparams, dtb, myname,
958	desthost, type)
959	except IOError, e:
960	raise service_error(service_error.internal,
961	"Failed to write config file for %s" % \
962	self.current_gateway)
963
964	gw_pubkey = "%s/keys/%s" % \
965	(self.tmpdir, self.gw_pubkey_base)
966	gw_secretkey = "%s/keys/%s" % \
967	(self.tmpdir, self.gw_secretkey_base)
968
969	pkfile = "%s/%s/%s" % \
970	( self.tmpdir, self.current_gateways,
971	self.gw_pubkey_base)
972	skfile = "%s/%s/%s" % \
973	( self.tmpdir, self.current_gateways,
974	self.gw_secretkey_base)
975
976	if not os.path.exists(pkfile):
977	try:
978	self.copy_file(gw_pubkey, pkfile)
979	except IOError:
980	service_error(service_error.internal,
981	"Failed to copy pubkey file")
982
983	if active and not os.path.exists(skfile):
984	try:
985	self.copy_file(gw_secretkey, skfile)
986	except IOError:
987	service_error(service_error.internal,
988	"Failed to copy secretkey file")
989	return True
990
991	class shunt_to_file:
992	def __init__(self, begin, end, filename):
993	self.begin = re.compile(begin)
994	self.end = re.compile(end)
995	self.in_shunt = False
996	self.file = None
997	self.filename = filename
998
999	def __call__(self, line):
1000	if not self.in_shunt:
1001	if self.begin.match(line):
1002	self.in_shunt = True
1003	try:
1004	self.file = open(self.filename, "w")
1005	except:
1006	self.file = None
1007	raise
1008	return True
1009	else:
1010	return False
1011	else:
1012	if self.end.match(line):
1013	if self.file:
1014	self.file.close()
1015	self.file = None
1016	self.in_shunt = False
1017	else:
1018	if self.file:
1019	print >>self.file, line
1020	return True
1021
1022	class shunt_to_list:
1023	def __init__(self, begin, end):
1024	self.begin = re.compile(begin)
1025	self.end = re.compile(end)
1026	self.in_shunt = False
1027	self.list = [ ]
1028
1029	def __call__(self, line):
1030	if not self.in_shunt:
1031	if self.begin.match(line):
1032	self.in_shunt = True
1033	return True
1034	else:
1035	return False
1036	else:
1037	if self.end.match(line):
1038	self.in_shunt = False
1039	else:
1040	self.list.append(line)
1041	return True
1042
1043	class shunt_to_string:
1044	def __init__(self, begin, end):
1045	self.begin = re.compile(begin)
1046	self.end = re.compile(end)
1047	self.in_shunt = False
1048	self.str = ""
1049
1050	def __call__(self, line):
1051	if not self.in_shunt:
1052	if self.begin.match(line):
1053	self.in_shunt = True
1054	return True
1055	else:
1056	return False
1057	else:
1058	if self.end.match(line):
1059	self.in_shunt = False
1060	else:
1061	self.str += line
1062	return True
1063
1064	def create_experiment(self, req, fid):
1065	try:
1066	tmpdir = tempfile.mkdtemp(prefix="split-")
1067	except IOError:
1068	raise service_error(service_error.internal, "Cannot create tmp dir")
1069
1070	gw_pubkey_base = "fed.%s.pub" % self.ssh_type
1071	gw_secretkey_base = "fed.%s" % self.ssh_type
1072	gw_pubkey = tmpdir + "/keys/" + gw_pubkey_base
1073	gw_secretkey = tmpdir + "/keys/" + gw_secretkey_base
1074	tclfile = tmpdir + "/experiment.tcl"
1075	tbparams = { }
1076
1077	pid = "dummy"
1078	gid = "dummy"
1079	eid = self.exp_stem
1080	if self.randomize_experiments:
1081	for i in range(0,5):
1082	eid += random.choice(string.ascii_letters)
1083	# XXX
1084	fail_soft = False
1085
1086	try:
1087	os.mkdir(tmpdir+"/keys")
1088	except OSError:
1089	raise service_error(service_error.internal,
1090	"Can't make temporary dir")
1091
1092	req = req.get('CreateRequestBody', None)
1093	if not req:
1094	raise service_error(service_error.req,
1095	"Bad request format (no CreateRequestBody)")
1096	# The tcl parser needs to read a file so put the content into that file
1097	file_content=req.get('experimentdescription', None)
1098	if file_content:
1099	try:
1100	f = open(tclfile, 'w')
1101	f.write(file_content)
1102	f.close()
1103	except IOError:
1104	raise service_error(service_error.internal,
1105	"Cannot write temp experiment description")
1106	else:
1107	raise service_error(service_error.req, "No experiment description")
1108
1109	try:
1110	self.generate_ssh_keys(gw_secretkey, self.ssh_type)
1111	except ValueError:
1112	raise service_error(service_error.server_config,
1113	"Bad key type (%s)" % self.ssh_type)
1114
1115	user = req.get('user', None)
1116	if user == None:
1117	raise service_error(service_error.req, "No user")
1118
1119	master = req.get('master', None)
1120	if master == None:
1121	raise service_error(service_error.req, "No master testbed label")
1122
1123
1124	tclcmd = [self.tclsh, self.tcl_splitter, '-s', '-x',
1125	str(self.muxmax), '-m', master, pid, gid, eid, tclfile]
1126	tclparser = Popen(tclcmd, stdout=PIPE)
1127
1128	allocated = { }
1129	started = { }
1130
1131	parse_current_testbed = self.current_testbed(eid, tmpdir)
1132	parse_allbeds = self.allbeds(self.get_access)
1133	parse_gateways = self.gateways(eid, master, tmpdir,
1134	gw_pubkey_base, gw_secretkey_base, self.copy_file)
1135	parse_vtopo = self.shunt_to_string("^#\s+Begin\s+Vtopo",
1136	"^#\s+End\s+Vtopo")
1137	parse_hostnames = self.shunt_to_file("^#\s+Begin\s+hostnames",
1138	"^#\s+End\s+hostnames", tmpdir + "/hosts")
1139	parse_tarfiles = self.shunt_to_list("^#\s+Begin\s+tarfiles",
1140	"^#\s+End\s+tarfiles")
1141	parse_rpms = self.shunt_to_list("^#\s+Begin\s+rpms",
1142	"^#\s+End\s+rpms")
1143
1144	for line in tclparser.stdout:
1145	line = line.rstrip()
1146	if parse_current_testbed(line, master, allocated, tbparams):
1147	continue
1148	elif parse_allbeds(line, user, tbparams):
1149	continue
1150	elif parse_gateways(line, allocated, tbparams):
1151	continue
1152	elif parse_vtopo(line):
1153	continue
1154	elif parse_hostnames(line):
1155	continue
1156	elif parse_tarfiles(line):
1157	continue
1158	elif parse_rpms(line):
1159	continue
1160	else:
1161	raise service_error(service_error.internal,
1162	"Bad tcl parse? %s" % line)
1163
1164	vtopo = self.gentopo(parse_vtopo.str)
1165	if not vtopo:
1166	raise service_error(service_error.internal,
1167	"Failed to generate virtual topology")
1168
1169	vis = self.genviz(vtopo)
1170	if not vis:
1171	raise service_error(service_error.internal,
1172	"Failed to generate visualization")
1173
1174	# Copy tarfiles and rpms needed at remote sites into a staging area
1175	try:
1176	for t in parse_tarfiles.list:
1177	if not os.path.exists("%s/tarfiles" % tmpdir):
1178	os.mkdir("%s/tarfiles" % tmpdir)
1179	self.copy_file(t, "%s/tarfiles/%s" % \
1180	(tmpdir, os.path.basename(t)))
1181	for r in parse_rpms.list:
1182	if not os.path.exists("%s/rpms" % tmpdir):
1183	os.mkdir("%s/rpms" % tmpdir)
1184	self.copy_file(r, "%s/rpms/%s" % \
1185	(tmpdir, os.path.basename(r)))
1186	except IOError, e:
1187	raise service_error(service_error.internal,
1188	"Cannot stage tarfile/rpm: %s" % e.strerror)
1189
1190	thread_pool_info = self.thread_pool()
1191	threads = [ ]
1192
1193	for tb in [ k for k in allocated.keys() if k != master]:
1194	# Wait until we have a free slot to start the next testbed load
1195	thread_pool_info.acquire()
1196	while thread_pool_info.started - \
1197	thread_pool_info.terminated >= self.nthreads:
1198	thread_pool_info.wait()
1199	thread_pool_info.release()
1200
1201	# Create and start a thread to start the segment, and save it to
1202	# get the return value later
1203	t = self.pooled_thread(target=self.start_segment,
1204	args=(tb, eid, tbparams, tmpdir, 0), name=tb,
1205	pdata=thread_pool_info, trace_file=self.trace_file)
1206	threads.append(t)
1207	t.start()
1208
1209	# Wait until all finish (the first clause of the while is to make sure
1210	# one starts)
1211	thread_pool_info.acquire()
1212	while thread_pool_info.started == 0 or \
1213	thread_pool_info.started > thread_pool_info.terminated:
1214	thread_pool_info.wait()
1215	thread_pool_info.release()
1216
1217	# If none failed, start the master
1218	failed = [ t.getName() for t in threads if not t.rv ]
1219
1220	if len(failed) == 0:
1221	if not self.start_segment(master, eid, tbparams, tmpdir):
1222	failed.append(master)
1223
1224	# If one failed clean up
1225	if len(failed) > 0:
1226	succeeded = [tb for tb in allocated.keys() if tb not in failed]
1227	if fail_soft:
1228	raise service_error(service_error.partial, \
1229	"Partial swap in on %s" % ",".join(succeeded))
1230	else:
1231	for tb in succeeded:
1232	self.stop_segment(tb, eid, tbparams)
1233	raise service_error(service_error.federant,
1234	"Swap in failed on %s" % ",".join(failed))
1235	else:
1236	if self.trace_file:
1237	print >>self.trace_file, "Experiment started"
1238
1239	# Generate an ID for the experiment (slice) and a certificate that the
1240	# allocator can use to prove they own it. We'll ship it back through
1241	# the encrypted connection.
1242	(expid, expcert) = generate_fedid("test", dir=tmpdir,
1243	trace=self.trace_file)
1244
1245	if self.trace_file:
1246	print >>self.trace_file, "removing %s" % tmpdir
1247
1248	# Walk up tmpdir, deleting as we go
1249	for path, dirs, files in os.walk(tmpdir, topdown=False):
1250	for f in files:
1251	os.remove(os.path.join(path, f))
1252	for d in dirs:
1253	os.rmdir(os.path.join(path, d))
1254	os.rmdir(tmpdir)
1255
1256	resp = { 'emulab' : [ tbparams[tb]['emulab'] \
1257	for tb in tbparams.keys() \
1258	if tbparams[tb].has_key('emulab') ],\
1259	'experiment': vtopo,\
1260	'vis' : vis,
1261	'experimentID' : { 'fedid': expid },\
1262	'experimentAccess': { 'X509' : expcert },\
1263	}
1264
1265	self.state[expid] = {\
1266	'emulab': [ tbparams[tb]['emulab'] \
1267	for tb in tbparams.keys() \
1268	if tbparams[tb].has_key('emulab') ],\
1269	'vis': vis,\
1270	'vtopo': vtopo,\
1271	}
1272
1273	return resp
1274
1275	def get_vtopo(self, req, fid):
1276
1277	req = req.get('VtopoRequestBody', None)
1278	if not req:
1279	raise service_error(service_error.req,
1280	"Bad request format (no VtopoRequestBody)")
1281	if req.has_key('experiment') and req['experiment'].has_key('fedid'):
1282	qfedid = req['experiment']['fedid']
1283	else:
1284	raise service_error(service_error.req, "Use fedid for lookup")
1285
1286	if self.state.has_key(qfedid):
1287	return { 'experiment' : {'fedid': qfedid },\
1288	'vtopo': self.state[qfedid]['vtopo'],\
1289	}
1290	else:
1291	raise service_error(service_error.req, "No such experiment")
1292
1293	def get_vis(self, req, fid):
1294
1295	req = req.get('VisRequestBody', None)
1296	if not req:
1297	raise service_error(service_error.req,
1298	"Bad request format (no VisRequestBody)")
1299	if req.has_key('experiment') and req['experiment'].has_key('fedid'):
1300	qfedid = req['experiment']['fedid']
1301	else:
1302	raise service_error(service_error.req, "Use fedid for lookup")
1303
1304	if self.state.has_key(qfedid):
1305	return { 'experiment' : {'fedid': qfedid },\
1306	'vis': self.state[qfedid]['vis'],\
1307	}
1308	else:
1309	raise service_error(service_error.req, "No such experiment")
1310
1311
1312	if __name__ == '__main__':
1313	from optparse import OptionParser
1314
1315	parser = OptionParser()
1316	parser.add_option('-d', '--debug', dest='debug', default=False,
1317	action='store_true', help='print actions rather than take them')
1318	parser.add_option('-f', '--file', dest='tcl', help='tcl file to parse')
1319	parser.add_option('-m', '--master', dest='master',
1320	help='testbed label for matster testbd')
1321	parser.add_option('-t', '--trace', dest='trace', default=None,
1322	help='file to print intermediate messages to')
1323	parser.add_option('-T', '--trace-stderr', dest='trace',
1324	action='store_const',const=sys.stderr,
1325	help='file to print intermediate messages to')
1326	opts, args = parser.parse_args()
1327
1328	trace_file = None
1329	if opts.trace:
1330	try:
1331	trace_file = open(opts.trace, 'w')
1332	except IOError:
1333	print >>sys.stderr, "Can't open trace file"
1334
1335	if opts.debug:
1336	if not trace_file:
1337	trace_file = sys.stderr
1338
1339	if opts.tcl != None:
1340	try:
1341	f = open(opts.tcl, 'r')
1342	content = ''.join(f)
1343	f.close()
1344	except IOError, e:
1345	sys.exit("Can't read %s: %s" % (opts.tcl, e))
1346	else:
1347	sys.exit("Must specify a file name")
1348
1349	if not opts.master:
1350	sys.exit("Must supply master tb label (--master)");
1351
1352	obj = fedd_create_experiment_local(
1353	debug=opts.debug,
1354	scripts_dir="/users/faber/testbed/federation",
1355	cert_file="./fedd_client.pem", cert_pwd="faber",
1356	ssh_pubkey_file='/users/faber/.ssh/id_rsa.pub',
1357	trusted_certs="./cacert.pem",
1358	tbmap = {
1359	'deter':'https://users.isi.deterlab.net:23235',
1360	'emulab':'https://users.isi.deterlab.net:23236',
1361	'ucb':'https://users.isi.deterlab.net:23237',
1362	},
1363	trace_file=trace_file
1364	)
1365	rv = obj.create_experiment( {\
1366	'experimentdescription' : content,
1367	'master' : opts.master,
1368	'user': [ {'userID' : { 'username' : 'faber' } } ],
1369	},
1370	None)
1371
1372	print rv

Note: See TracBrowser for help on using the repository browser.

Download in other formats: