#!/usr/local/bin/python import os,sys import stat # for chmod constants import re import random import string import copy import pickle import logging import subprocess import traceback import socket from threading import * from M2Crypto.SSL import SSLError from access import access_base from util import * from deter import fedid, generate_fedid from authorizer import authorizer, abac_authorizer from service_error import service_error from remote_service import xmlrpc_handler, soap_handler, service_caller from proof import proof as access_proof import httplib import tempfile from urlparse import urlparse from deter import topdl import list_log import emulab_segment import emulab_shared_nat_segment # Make log messages disappear if noone configures a fedd logger class nullHandler(logging.Handler): def emit(self, record): pass fl = logging.getLogger("fedd.access") fl.addHandler(nullHandler()) class access(access_base): """ The implementation of access control based on mapping users to projects. Users can be mapped to existing projects or have projects created dynamically. This implements both direct requests and proxies. """ max_name_len = 19 def __init__(self, config=None, auth=None): """ Initializer. Pulls parameters out of the ConfigParser's access section. """ access_base.__init__(self, config, auth) self.max_name_len = access.max_name_len self.allow_proxy = config.getboolean("access", "allow_proxy") self.domain = config.get("access", "domain") self.userconfdir = config.get("access","userconfdir") self.userconfcmd = config.get("access","userconfcmd") self.userconfurl = config.get("access","userconfurl") self.federation_software = config.get("access", "federation_software") self.portal_software = config.get("access", "portal_software") self.local_seer_software = config.get("access", "local_seer_software") self.local_seer_image = config.get("access", "local_seer_image") self.local_seer_start = config.get("access", "local_seer_start") self.seer_master_start = config.get("access", "seer_master_start") self.ssh_privkey_file = config.get("access","ssh_privkey_file") self.ssh_pubkey_file = config.get("access","ssh_pubkey_file") self.ssh_port = config.get("access","ssh_port") or "22" self.boss = config.get("access", "boss") self.ops = config.get("access", "ops") self.xmlrpc_cert = config.get("access", "xmlrpc_cert") self.xmlrpc_certpw = config.get("access", "xmlrpc_certpw") self.dragon_endpoint = config.get("access", "dragon") self.dragon_vlans = config.get("access", "dragon_vlans") self.deter_internal = config.get("access", "deter_internal") self.tunnel_config = config.getboolean("access", "tunnel_config") self.portal_command = config.get("access", "portal_command") self.portal_image = config.get("access", "portal_image") self.portal_type = config.get("access", "portal_type") or "pc" self.portal_startcommand = config.get("access", "portal_startcommand") self.node_startcommand = config.get("access", "node_startcommand") self.nat_portal = config.get("access", "nat_portal") self.shared_nat = config.getboolean("access", "shared_nat") self.uri = 'https://%s:%d' % (socket.getfqdn(), self.get_port(config.get("globals", "services", "23235"))) self.federation_software = self.software_list(self.federation_software) self.portal_software = self.software_list(self.portal_software) self.local_seer_software = self.software_list(self.local_seer_software) self.access_type = self.access_type.lower() if self.shared_nat: self.start_segment = emulab_shared_nat_segment.start_segment self.stop_segment = emulab_shared_nat_segment.stop_segment self.info_segment = emulab_shared_nat_segment.info_segment self.operation_segment = emulab_shared_nat_segment.operation_segment self.exports_segment = emulab_shared_nat_segment.exports_segment else: self.start_segment = emulab_segment.start_segment self.stop_segment = emulab_segment.stop_segment self.info_segment = emulab_segment.info_segment self.operation_segment = emulab_segment.operation_segment self.exports_segment = emulab_segment.exports_segment self.restricted = [ ] tb = config.get('access', 'testbed') if tb: self.testbed = [ t.strip() for t in tb.split(',') ] else: self.testbed = [ ] # authorization information self.auth_type = config.get('access', 'auth_type') \ or 'abac' self.auth_dir = config.get('access', 'auth_dir') accessdb = config.get("access", "accessdb") # initialize the authorization system if self.auth_type == 'abac': self.auth = abac_authorizer(load=self.auth_dir) self.access = [ ] if accessdb: self.read_access(accessdb, self.access_tuple) else: raise service_error(service_error.internal, "Unknown auth_type: %s" % self.auth_type) # read_state in the base_class self.state_lock.acquire() if 'allocation' not in self.state: self.state['allocation']= { } self.allocation = self.state['allocation'] self.state_lock.release() self.exports = { 'SMB': self.export_SMB, 'seer': self.export_seer, 'tmcd': self.export_tmcd, 'userconfig': self.export_userconfig, 'project_export': self.export_project_export, 'local_seer_control': self.export_local_seer, 'seer_master': self.export_seer_master, 'hide_hosts': self.export_hide_hosts, } if not self.local_seer_image or not self.local_seer_software or \ not self.local_seer_start: if 'local_seer_control' in self.exports: del self.exports['local_seer_control'] if not self.local_seer_image or not self.local_seer_software or \ not self.seer_master_start: if 'seer_master' in self.exports: del self.exports['seer_master'] self.soap_services = {\ 'RequestAccess': soap_handler("RequestAccess", self.RequestAccess), 'ReleaseAccess': soap_handler("ReleaseAccess", self.ReleaseAccess), 'StartSegment': soap_handler("StartSegment", self.StartSegment), 'TerminateSegment': soap_handler("TerminateSegment", self.TerminateSegment), 'InfoSegment': soap_handler("InfoSegment", self.InfoSegment), 'OperationSegment': soap_handler("OperationSegment", self.OperationSegment), } self.xmlrpc_services = {\ 'RequestAccess': xmlrpc_handler('RequestAccess', self.RequestAccess), 'ReleaseAccess': xmlrpc_handler('ReleaseAccess', self.ReleaseAccess), 'StartSegment': xmlrpc_handler("StartSegment", self.StartSegment), 'TerminateSegment': xmlrpc_handler('TerminateSegment', self.TerminateSegment), 'InfoSegment': xmlrpc_handler("InfoSegment", self.InfoSegment), 'OperationSegment': xmlrpc_handler("OperationSegment", self.OperationSegment), } self.call_SetValue = service_caller('SetValue', log=self.log) self.call_GetValue = service_caller('GetValue', log=self.log) @staticmethod def get_port(ps): ''' Take a fedd service string and return the first port. Used in creating the testbed uri identifier. ''' p = ps.split(',') smallport = p[0].split(':') try: rv = int(smallport[0]) except ValueError: rv = 23235 return rv @staticmethod def access_tuple(str): """ Convert a string of the form (id, id, id) into an access_project. This is called by read_access to convert to local attributes. It returns a tuple of the form (project, user, certificate_file). """ str = str.strip() if str.startswith('(') and str.endswith(')') and str.count(',') == 2: # The slice takes the parens off the string. proj, user, cert = str[1:-1].split(',') return (proj.strip(), user.strip(), cert.strip()) else: raise self.parse_error( 'Bad mapping (unbalanced parens or more than 2 commas)') # RequestAccess support routines def save_project_state(self, aid, pname, uname, certf, owners): """ Save the project, user, and owners associated with this allocation. This creates the allocation entry. """ self.state_lock.acquire() self.allocation[aid] = { } self.allocation[aid]['project'] = pname self.allocation[aid]['user'] = uname self.allocation[aid]['cert'] = certf self.allocation[aid]['owners'] = owners self.write_state() self.state_lock.release() return (pname, uname) # End of RequestAccess support routines def RequestAccess(self, req, fid): """ Handle the access request. Proxy if not for us. Parse out the fields and make the allocations or rejections if for us, otherwise, assuming we're willing to proxy, proxy the request out. """ def gateway_hardware(h): if h == 'GWTYPE': return self.portal_type or 'GWTYPE' else: return h def get_export_project(svcs): """ if the service requests includes one to export a project, return that project. """ rv = None for s in svcs: if s.get('name', '') == 'project_export' and \ s.get('visibility', '') == 'export': if not rv: for a in s.get('fedAttr', []): if a.get('attribute', '') == 'project' \ and 'value' in a: rv = a['value'] else: raise service_error(service_error, access, 'Requesting multiple project exports is ' + \ 'not supported'); return rv self.log.info("RequestAccess called by %s" % fid) # The dance to get into the request body if req.has_key('RequestAccessRequestBody'): req = req['RequestAccessRequestBody'] else: raise service_error(service_error.req, "No request!?") # if this includes a project export request, construct a filter such # that only the ABAC attributes mapped to that project are checked for # access. if 'service' in req: ep = get_export_project(req['service']) if ep: pf = lambda(a): a.value[0] == ep else: pf = None else: ep = None pf = None if self.auth.import_credentials( data_list=req.get('abac_credential', [])): self.auth.save() else: self.log.debug('failed to import incoming credentials') if self.auth_type == 'abac': found, owners, proof = self.lookup_access(req, fid, filter=pf) else: raise service_error(service_error.internal, 'Unknown auth_type: %s' % self.auth_type) ap = None # keep track of what's been added allocID, alloc_cert = generate_fedid(subj="alloc", log=self.log) aid = unicode(allocID) pname, uname = self.save_project_state(aid, found[0], found[1], found[2], owners) services, svc_state = self.export_services(req.get('service',[]), pname, uname) self.state_lock.acquire() # Store services state in global state for k, v in svc_state.items(): self.allocation[aid][k] = v self.append_allocation_authorization(aid, set([(o, allocID) for o in owners]), state_attr='allocation') self.write_state() self.state_lock.release() try: f = open("%s/%s.pem" % (self.certdir, aid), "w") print >>f, alloc_cert f.close() except EnvironmentError, e: self.log.info("RequestAccess failed for by %s: internal error" \ % fid) raise service_error(service_error.internal, "Can't open %s/%s : %s" % (self.certdir, aid, e)) self.log.debug('RequestAccess Returning allocation ID: %s' % allocID) resp = self.build_access_response({ 'fedid': allocID } , pname, services, proof) return resp def ReleaseAccess(self, req, fid): self.log.info("ReleaseAccess called by %s" % fid) # The dance to get into the request body if req.has_key('ReleaseAccessRequestBody'): req = req['ReleaseAccessRequestBody'] else: raise service_error(service_error.req, "No request!?") try: if req['allocID'].has_key('localname'): auth_attr = aid = req['allocID']['localname'] elif req['allocID'].has_key('fedid'): aid = unicode(req['allocID']['fedid']) auth_attr = req['allocID']['fedid'] else: raise service_error(service_error.req, "Only localnames and fedids are understood") except KeyError: raise service_error(service_error.req, "Badly formed request") self.log.debug("[access] deallocation requested for %s by %s" % \ (aid, fid)) access_ok, proof = self.auth.check_attribute(fid, auth_attr, with_proof=True) if not access_ok: self.log.debug("[access] deallocation denied for %s", aid) raise service_error(service_error.access, "Access Denied") self.state_lock.acquire() if aid in self.allocation: self.log.debug("Found allocation for %s" %aid) self.clear_allocation_authorization(aid, state_attr='allocation') del self.allocation[aid] self.write_state() self.state_lock.release() # Remove the access cert cf = "%s/%s.pem" % (self.certdir, aid) self.log.debug("Removing %s" % cf) os.remove(cf) self.log.info("ReleaseAccess succeeded for %s" % fid) return { 'allocID': req['allocID'], 'proof': proof.to_dict() } else: self.state_lock.release() raise service_error(service_error.req, "No such allocation") # These are subroutines for StartSegment def generate_ns2(self, topo, expfn, softdir, connInfo): """ Convert topo into an ns2 file, decorated with appropriate commands for the particular testbed setup. Convert all requests for software, etc to point at the staged copies on this testbed and add the federation startcommands. """ class dragon_commands: """ Functor to spit out approrpiate dragon commands for nodes listed in the connectivity description. The constructor makes a dict mapping dragon nodes to their parameters and the __call__ checks each element in turn for membership. """ def __init__(self, map): self.node_info = map def __call__(self, e): s = "" if isinstance(e, topdl.Computer): if self.node_info.has_key(e.name): info = self.node_info[e.name] for ifname, vlan, type in info: for i in e.interface: if i.name == ifname: addr = i.get_attribute('ip4_address') subs = i.substrate[0] break else: raise service_error(service_error.internal, "No interface %s on element %s" % \ (ifname, e.name)) # XXX: do netmask right if type =='link': s = ("tb-allow-external ${%s} " + \ "dragonportal ip %s vlan %s " + \ "netmask 255.255.255.0\n") % \ (topdl.to_tcl_name(e.name), addr, vlan) elif type =='lan': s = ("tb-allow-external ${%s} " + \ "dragonportal " + \ "ip %s vlan %s usurp %s\n") % \ (topdl.to_tcl_name(e.name), addr, vlan, subs) else: raise service_error(service_error_internal, "Unknown DRAGON type %s" % type) return s class not_dragon: """ Return true if a node is in the given map of dragon nodes. """ def __init__(self, map): self.nodes = set(map.keys()) def __call__(self, e): return e.name not in self.nodes def have_portals(top): ''' Return true if the topology has a portal node ''' # The else is on the for for e in top.elements: if isinstance(e, topdl.Computer) and e.get_attribute('portal'): return True else: return False # Main line of generate_ns2 t = topo.clone() # Create the map of nodes that need direct connections (dragon # connections) from the connInfo dragon_map = { } for i in [ i for i in connInfo if i['type'] == 'transit']: for a in i.get('fedAttr', []): if a['attribute'] == 'vlan_id': vlan = a['value'] break else: raise service_error(service_error.internal, "No vlan tag") members = i.get('member', []) if len(members) > 1: type = 'lan' else: type = 'link' try: for m in members: if m['element'] in dragon_map: dragon_map[m['element']].append(( m['interface'], vlan, type)) else: dragon_map[m['element']] = [( m['interface'], vlan, type),] except KeyError: raise service_error(service_error.req, "Missing connectivity info") def output_fixed_filter(e): if not isinstance(e, topdl.Computer): return "" fn = e.get_attribute('fixed') if fn is None: return "" else: return 'tb-fix-node ${%s} %s\n' % (topdl.to_tcl_name(e.name), fn) def output_external_access(e): ''' Allow nodes that have explicit external access requests attached to have external access. This will be replaced by a real risky experiment filter. ''' if not isinstance(e, topdl.Computer): return "" ex = e.get_attribute('containers:external') if ex is None: return "" else: return 'tb-allow-external ${%s}\n' % topdl.to_tcl_name(e.name) # Weed out the things we aren't going to instantiate: Segments, portal # substrates, and portal interfaces. (The copy in the for loop allows # us to delete from e.elements in side the for loop). While we're # touching all the elements, we also adjust paths from the original # testbed to local testbed paths and put the federation commands into # the start commands local = len(dragon_map) == 0 and not have_portals(t) if local: routing = 'Static' else: routing = 'Manual' if local: self.log.debug("Local experiment.") for e in [e for e in t.elements]: if isinstance(e, topdl.Segment): t.elements.remove(e) if isinstance(e, topdl.Computer): self.add_kit(e, self.federation_software) if e.get_attribute('portal') and self.portal_startcommand: # Add local portal support software self.add_kit(e, self.portal_software) # Portals never have a user-specified start command e.set_attribute('startup', self.portal_startcommand) elif not local and self.node_startcommand: if e.get_attribute('startup'): e.set_attribute('startup', "%s \\$USER '%s'" % \ (self.node_startcommand, e.get_attribute('startup'))) else: e.set_attribute('startup', self.node_startcommand) dinf = [i[0] for i in dragon_map.get(e.name, []) ] # Remove portal interfaces that do not connect to DRAGON e.interface = [i for i in e.interface \ if not i.get_attribute('portal') or i.name in dinf ] # Fix software paths for s in getattr(e, 'software', []): s.location = re.sub("^.*/", softdir, s.location) t.substrates = [ s.clone() for s in t.substrates ] t.incorporate_elements() # Customize the ns2 output for local portal commands and images filters = [output_fixed_filter, output_external_access] if self.dragon_endpoint: add_filter = not_dragon(dragon_map) filters.append(dragon_commands(dragon_map)) else: add_filter = None if self.portal_command: if self.shared_nat: suffix = 'shared nat 0/0/tcp rdr 22/tcp' else: suffix = '' filters.append(topdl.generate_portal_command_filter( self.portal_command, add_filter=add_filter, suffix=suffix)) if self.portal_image: filters.append(topdl.generate_portal_image_filter( self.portal_image)) if self.portal_type: filters.append(topdl.generate_portal_hardware_filter( self.portal_type)) # Convert to ns and write it out expfile = topdl.topology_to_ns2(t, filters, routing=routing) try: f = open(expfn, "w") print >>f, expfile f.close() except EnvironmentError: raise service_error(service_error.internal, "Cannot write experiment file %s: %s" % (expfn,e)) def export_store_info(self, cf, proj, ename, connInfo): """ For the export requests in the connection info, send out teh SetValue calls for parameters initialized by the segment's export_segment object. Values are all in the connInfo parameter's value fields. """ for c in connInfo: for p in c.get('parameter', []): # Ignore non-output parameters if p.get('type', '') != 'output': continue # Sanity check store, key and value surl = p.get('store', None) key = p.get('key', None) value = p.get('value', None) if surl is None: self.log.error("Export parameter without store: %s" % \ p.get('name')) continue if key is None: self.log.error("Export parameter without key: %s" % \ p.get('name')) continue if value is None: self.log.error("Unknown (unset) export parameter: %s" % \ p.get('name')) continue # Set the store value req = { 'name': key, 'value': value } self.call_SetValue(surl, req, cf) def add_seer_node(self, topo, name, startup): """ Add a seer node to the given topology, with the startup command passed in. Used by configure seer_services. """ c_node = topdl.Computer( name=name, os= topdl.OperatingSystem( attribute=[ { 'attribute': 'osid', 'value': self.local_seer_image }, ]), attribute=[ { 'attribute': 'startup', 'value': startup }, ] ) self.add_kit(c_node, self.local_seer_software) topo.elements.append(c_node) def configure_seer_services(self, services, topo, softdir): """ Make changes to the topology required for the seer requests being made. Specifically, add any control or master nodes required and set up the start commands on the nodes to interconnect them. """ local_seer = False # True if we need to add a control node collect_seer = False # True if there is a seer-master node seer_master= False # True if we need to add the seer-master for s in services: s_name = s.get('name', '') s_vis = s.get('visibility','') if s_name == 'local_seer_control' and s_vis == 'export': local_seer = True elif s_name == 'seer_master': if s_vis == 'import': collect_seer = True elif s_vis == 'export': seer_master = True # We've got the whole picture now, so add nodes if needed and configure # them to interconnect properly. if local_seer or seer_master: # Copy local seer control node software to the tempdir for l, f in self.local_seer_software: base = os.path.basename(f) copy_file(f, "%s/%s" % (softdir, base)) # If we're collecting seers somewhere the controllers need to talk to # the master. In testbeds that export the service, that will be a # local node that we'll add below. Elsewhere it will be the control # portal that will port forward to the exporting master. if local_seer: if collect_seer: startup = "%s -C %s" % (self.local_seer_start, "seer-master") else: startup = self.local_seer_start self.add_seer_node(topo, 'control', startup) # If this is the seer master, add that node, too. if seer_master: self.add_seer_node(topo, 'seer-master', "%s -R -n -R seer-master -R -A -R sink" % \ self.seer_master_start) def retrieve_software(self, topo, certfile, softdir): """ Collect the software that nodes in the topology need loaded and stage it locally. This implies retrieving it from the experiment_controller and placing it into softdir. Certfile is used to prove that this node has access to that data (it's the allocation/segment fedid). Finally local portal and federation software is also copied to the same staging directory for simplicity - all software needed for experiment creation is in softdir. """ sw = set() for e in topo.elements: for s in getattr(e, 'software', []): sw.add(s.location) for s in sw: self.log.debug("Retrieving %s" % s) try: get_url(s, certfile, softdir, log=self.log) except: t, v, st = sys.exc_info() raise service_error(service_error.internal, "Error retrieving %s: %s" % (s, v)) # Copy local federation and portal node software to the tempdir for s in (self.federation_software, self.portal_software): for l, f in s: base = os.path.basename(f) copy_file(f, "%s/%s" % (softdir, base)) def initialize_experiment_info(self, attrs, aid, certfile, tmpdir): """ Gather common configuration files, retrieve or create an experiment name and project name, and return the ssh_key filenames. Create an allocation log bound to the state log variable as well. """ configs = ('hosts', 'ssh_pubkey', 'ssh_secretkey', 'seer_ca_pem', 'seer_node_pem', 'route.tgz') ename = None pubkey_base = None secretkey_base = None proj = None user = None alloc_log = None nonce_experiment = False vchars_re = '[^' + string.ascii_letters + string.digits + '-]' self.state_lock.acquire() if aid in self.allocation: proj = self.allocation[aid].get('project', None) self.state_lock.release() if not proj: raise service_error(service_error.internal, "Can't find project for %s" %aid) for a in attrs: if a['attribute'] in configs: try: self.log.debug("Retrieving %s from %s" % \ (a['attribute'], a['value'])) get_url(a['value'], certfile, tmpdir, log=self.log) except: t, v, st = sys.exc_info() raise service_error(service_error.internal, "Error retrieving %s: %s" % (a.get('value', ""), v)) if a['attribute'] == 'ssh_pubkey': pubkey_base = a['value'].rpartition('/')[2] if a['attribute'] == 'ssh_secretkey': secretkey_base = a['value'].rpartition('/')[2] if a['attribute'] == 'experiment_name': ename = a['value'] # Names longer than the emulab max are discarded if ename and len(ename) <= self.max_name_len: # Clean up the experiment name so that emulab will accept it. ename = re.sub(vchars_re, '-', ename) else: ename = "" for i in range(0,5): ename += random.choice(string.ascii_letters) nonce_experiment = True self.log.warn("No experiment name or suggestion too long: " + \ "picked one randomly: %s" % ename) if not pubkey_base: raise service_error(service_error.req, "No public key attribute") if not secretkey_base: raise service_error(service_error.req, "No secret key attribute") self.state_lock.acquire() if aid in self.allocation: user = self.allocation[aid].get('user', None) cert = self.allocation[aid].get('cert', None) self.allocation[aid]['experiment'] = ename self.allocation[aid]['nonce'] = nonce_experiment self.allocation[aid]['log'] = [ ] # Create a logger that logs to the experiment's state object as # well as to the main log file. alloc_log = logging.getLogger('fedd.access.%s' % ename) h = logging.StreamHandler( list_log.list_log(self.allocation[aid]['log'])) # XXX: there should be a global one of these rather than # repeating the code. h.setFormatter(logging.Formatter( "%(asctime)s %(name)s %(message)s", '%d %b %y %H:%M:%S')) alloc_log.addHandler(h) self.write_state() self.state_lock.release() if not user: raise service_error(service_error.internal, "Can't find creation user for %s" %aid) return (ename, proj, user, cert, pubkey_base, secretkey_base, alloc_log) def decorate_topology(self, info, t): """ Copy the physical mapping and status onto the topology. Used by StartSegment and InfoSegment """ def add_new(ann, attr): for a in ann: if a not in attr: attr.append(a) def merge_os(os, e): if len(e.os) == 0: # No OS at all: if os.get_attribute('emulab_access:image'): os.set_attribute('emulab_access:initial_image', os.get_attribute('emulab_access:image')) e.os = [ os ] elif len(e.os) == 1: # There's one OS, copy the initial image and replace eos = e.os[0] initial = eos.get_attribute('emulab_access:initial_image') if initial: os.set_attribute('emulab_access:initial_image', initial) e.os = [ os] else: # Multiple OSes, replace or append for eos in e.os: if os.name == eos.name: eos.version = os.version eos.version = os.distribution eos.version = os.distributionversion for a in os.attribute: if eos.get_attribute(a.attribute): eos.remove_attribute(a.attribute) eos.set_attribute(a.attribute, a.value) break else: e.os.append(os) if t is None: return i = 0 # For fake debugging instantiation # Copy the assigned names into the return topology for e in t.elements: if isinstance(e, topdl.Computer): if not self.create_debug: if e.name in info.node: add_new(("%s%s" % (info.node[e.name].pname, self.domain),), e.localname) add_new(("%s%s" % (info.node[e.name].lname, self.domain),), e.localname) e.status = info.node[e.name].status os = info.node[e.name].getOS() if os: merge_os(os, e) else: # Simple debugging assignment add_new(("node%d%s" % (i, self.domain),), e.localname) e.status = 'active' add_new(('testop1', 'testop2'), e.operation) i += 1 for s in t.substrates: if s.name in info.subs: sub = info.subs[s.name] if sub.cap is not None: s.capacity = topdl.Capacity(sub.cap, 'max') if sub.delay is not None: s.delay = topdl.Latency(sub.delay, 'max') # XXX interfaces def finalize_experiment(self, starter, topo, aid, alloc_id, proof): """ Store key bits of experiment state in the global repository, including the response that may need to be replayed, and return the response. """ def get_localnames(t): names = [ ] for e in t.elements: if isinstance(e, topdl.Computer): n = e.get_attribute('testbed') if n is not None and n not in names: names.append(n) return names i = 0 t = topo.clone() self.decorate_topology(starter, t) # Grab the log (this is some anal locking, but better safe than # sorry) self.state_lock.acquire() # Put information about this testbed into the topdl tb = topdl.Testbed(self.uri, "deter", localname=get_localnames(t), attribute=[ { 'attribute': 'project', 'value': self.allocation[aid]['project'] }, { 'attribute': 'experiment', 'value': self.allocation[aid]['experiment'] }]) t.elements.append(tb) logv = "".join(self.allocation[aid]['log']) # It's possible that the StartSegment call gets retried (!). # if the 'started' key is in the allocation, we'll return it rather # than redo the setup. self.allocation[aid]['started'] = { 'allocID': alloc_id, 'allocationLog': logv, 'segmentdescription': { 'topdldescription': t.to_dict() }, 'proof': proof.to_dict(), } self.allocation[aid]['topo'] = t retval = copy.copy(self.allocation[aid]['started']) self.write_state() self.state_lock.release() return retval # End of StartSegment support routines def StartSegment(self, req, fid): err = None # Any service_error generated after tmpdir is created rv = None # Return value from segment creation self.log.info("StartSegment called by %s" % fid) try: req = req['StartSegmentRequestBody'] auth_attr = req['allocID']['fedid'] topref = req['segmentdescription']['topdldescription'] except KeyError: raise service_error(server_error.req, "Badly formed request") connInfo = req.get('connection', []) services = req.get('service', []) aid = "%s" % auth_attr attrs = req.get('fedAttr', []) access_ok, proof = self.auth.check_attribute(fid, auth_attr, with_proof=True) if not access_ok: self.log.info("StartSegment for %s failed: access denied" % fid) raise service_error(service_error.access, "Access denied") else: # See if this is a replay of an earlier succeeded StartSegment - # sometimes SSL kills 'em. If so, replay the response rather than # redoing the allocation. self.state_lock.acquire() retval = self.allocation[aid].get('started', None) self.state_lock.release() if retval: self.log.warning("Duplicate StartSegment for %s: " % aid + \ "replaying response") return retval # A new request. Do it. if topref: topo = topdl.Topology(**topref) else: raise service_error(service_error.req, "Request missing segmentdescription'") certfile = "%s/%s.pem" % (self.certdir, auth_attr) try: tmpdir = tempfile.mkdtemp(prefix="access-") softdir = "%s/software" % tmpdir os.mkdir(softdir) except EnvironmentError: self.log.info("StartSegment for %s failed: internal error" % fid) raise service_error(service_error.internal, "Cannot create tmp dir") # Try block alllows us to clean up temporary files. try: self.retrieve_software(topo, certfile, softdir) ename, proj, user, xmlrpc_cert, pubkey_base, secretkey_base, \ alloc_log = self.initialize_experiment_info(attrs, aid, certfile, tmpdir) # A misconfigured cert in the ABAC map can be confusing... if not os.access(xmlrpc_cert, os.R_OK): self.log.error("Cannot open user's emulab SSL cert: %s" % \ xmlrpc_cert) raise service_error(service_error.internal, "Cannot open user's emulab SSL cert: %s" % xmlrpc_cert) if '/' in proj: proj, gid = proj.split('/') else: gid = None # Set up userconf and seer if needed self.configure_userconf(services, tmpdir) self.configure_seer_services(services, topo, softdir) # Get and send synch store variables exporter = self.exports_segment(keyfile=self.ssh_privkey_file, debug=self.create_debug, log=alloc_log, boss=self.boss, ops=self.ops, cert=xmlrpc_cert) if not exporter(self, ename, proj, user, connInfo, tmpdir, gid): raise service_error(service_error.internal, 'Failed to gather export variables') self.export_store_info(certfile, proj, ename, connInfo) self.import_store_info(certfile, connInfo) expfile = "%s/experiment.tcl" % tmpdir self.generate_portal_configs(topo, pubkey_base, secretkey_base, tmpdir, proj, ename, connInfo, services) self.generate_ns2(topo, expfile, "/proj/%s/software/%s/" % (proj, ename), connInfo) starter = self.start_segment(keyfile=self.ssh_privkey_file, debug=self.create_debug, log=alloc_log, boss=self.boss, ops=self.ops, cert=xmlrpc_cert) rv = starter(self, ename, proj, user, expfile, tmpdir, gid=gid) except service_error, e: self.log.info("StartSegment for %s failed: %s" % (fid, e)) err = e except: t, v, st = sys.exc_info() self.log.info("StartSegment for %s failed:unexpected error: %s" \ % (fid, traceback.extract_tb(st))) err = service_error(service_error.internal, "%s: %s" % \ (v, traceback.extract_tb(st))) # Walk up tmpdir, deleting as we go if self.cleanup: self.remove_dirs(tmpdir) else: self.log.debug("[StartSegment]: not removing %s" % tmpdir) if rv: self.log.info("StartSegment for %s succeeded" % fid) return self.finalize_experiment(starter, topo, aid, req['allocID'], proof) elif err: raise service_error(service_error.federant, "Swapin failed: %s" % err) else: raise service_error(service_error.federant, "Swapin failed") def TerminateSegment(self, req, fid): self.log.info("TerminateSegment called by %s" % fid) try: req = req['TerminateSegmentRequestBody'] except KeyError: raise service_error(server_error.req, "Badly formed request") auth_attr = req['allocID']['fedid'] aid = "%s" % auth_attr attrs = req.get('fedAttr', []) access_ok, proof = self.auth.check_attribute(fid, auth_attr, with_proof=True) if not access_ok: raise service_error(service_error.access, "Access denied") self.state_lock.acquire() if aid in self.allocation: proj = self.allocation[aid].get('project', None) user = self.allocation[aid].get('user', None) xmlrpc_cert = self.allocation[aid].get('cert', None) ename = self.allocation[aid].get('experiment', None) nonce = self.allocation[aid].get('nonce', False) else: proj = None user = None ename = None nonce = False xmlrpc_cert = None self.state_lock.release() if not proj: self.log.info("TerminateSegment failed for %s: cannot find project"\ % fid) raise service_error(service_error.internal, "Can't find project for %s" % aid) else: if '/' in proj: proj, gid = proj.split('/') else: gid = None if not user: self.log.info("TerminateSegment failed for %s: cannot find user"\ % fid) raise service_error(service_error.internal, "Can't find creation user for %s" % aid) if not ename: self.log.info( "TerminateSegment failed for %s: cannot find experiment"\ % fid) raise service_error(service_error.internal, "Can't find experiment name for %s" % aid) stopper = self.stop_segment(keyfile=self.ssh_privkey_file, debug=self.create_debug, boss=self.boss, ops=self.ops, cert=xmlrpc_cert) stopper(self, user, proj, ename, gid, nonce) self.log.info("TerminateSegment succeeded for %s %s %s" % \ (fid, proj, ename)) self.state_lock.acquire() # Remove the started flag/info - the segment is no longer started if aid in self.allocation: if 'started' in self.allocation[aid]: del self.allocation[aid]['started'] self.write_state() self.state_lock.release() return { 'allocID': req['allocID'], 'proof': proof.to_dict() } def InfoSegment(self, req, fid): self.log.info("InfoSegment called by %s" % fid) try: req = req['InfoSegmentRequestBody'] except KeyError: raise service_error(server_error.req, "Badly formed request") auth_attr = req['allocID']['fedid'] aid = "%s" % auth_attr access_ok, proof = self.auth.check_attribute(fid, auth_attr, with_proof=True) if not access_ok: raise service_error(service_error.access, "Access denied") self.state_lock.acquire() if aid in self.allocation: topo = self.allocation[aid].get('topo', None) if topo: topo = topo.clone() proj = self.allocation[aid].get('project', None) user = self.allocation[aid].get('user', None) xmlrpc_cert = self.allocation[aid].get('cert', None) ename = self.allocation[aid].get('experiment', None) else: proj = None user = None ename = None topo = None xmlrpc_cert = None self.state_lock.release() if not proj: self.log.info("InfoSegment failed for %s: cannot find project"% fid) raise service_error(service_error.internal, "Can't find project for %s" % aid) else: if '/' in proj: proj, gid = proj.split('/') else: gid = None if not user: self.log.info("InfoSegment failed for %s: cannot find user"% fid) raise service_error(service_error.internal, "Can't find creation user for %s" % aid) if not ename: self.log.info("InfoSegment failed for %s: cannot find exp"% fid) raise service_error(service_error.internal, "Can't find experiment name for %s" % aid) info = self.info_segment(keyfile=self.ssh_privkey_file, debug=self.create_debug, boss=self.boss, ops=self.ops, cert=xmlrpc_cert) info(self, user, proj, ename) self.log.info("InfoSegment gathered info for %s %s %s %s" % \ (fid, user, proj, ename)) self.decorate_topology(info, topo) self.state_lock.acquire() if aid in self.allocation: self.allocation[aid]['topo'] = topo self.write_state() self.state_lock.release() self.log.info("InfoSegment updated info for %s %s %s %s" % \ (fid, user, proj, ename)) rv = { 'allocID': req['allocID'], 'proof': proof.to_dict(), } self.log.info("InfoSegment succeeded info for %s %s %s %s" % \ (fid, user, proj, ename)) if topo: rv['segmentdescription'] = { 'topdldescription' : topo.to_dict() } return rv def OperationSegment(self, req, fid): def get_pname(e): """ Get the physical name of a node """ if e.localname: return re.sub('\..*','', e.localname[0]) else: return None self.log.info("OperationSegment called by %s" % fid) try: req = req['OperationSegmentRequestBody'] except KeyError: raise service_error(server_error.req, "Badly formed request") auth_attr = req['allocID']['fedid'] aid = "%s" % auth_attr access_ok, proof = self.auth.check_attribute(fid, auth_attr, with_proof=True) if not access_ok: self.log.info("OperationSegment failed for %s: access denied" % fid) raise service_error(service_error.access, "Access denied") op = req.get('operation', None) targets = req.get('target', None) params = req.get('parameter', None) if op is None : self.log.info("OperationSegment failed for %s: no operation" % fid) raise service_error(service_error.req, "missing operation") elif targets is None: self.log.info("OperationSegment failed for %s: no targets" % fid) raise service_error(service_error.req, "no targets") self.state_lock.acquire() if aid in self.allocation: topo = self.allocation[aid].get('topo', None) if topo: topo = topo.clone() xmlrpc_cert = self.allocation[aid].get('cert', None) else: topo = None xmlrpc_cert = None self.state_lock.release() targets = copy.copy(targets) ptargets = { } for e in topo.elements: if isinstance(e, topdl.Computer): if e.name in targets: targets.remove(e.name) pn = get_pname(e) if pn: ptargets[e.name] = pn status = [ operation_status(t, operation_status.no_target) \ for t in targets] ops = self.operation_segment(keyfile=self.ssh_privkey_file, debug=self.create_debug, boss=self.boss, ops=self.ops, cert=xmlrpc_cert) ops(self, op, ptargets, params, topo) self.log.info("OperationSegment operated for %s" % fid) status.extend(ops.status) self.log.info("OperationSegment succeed for %s" % fid) return { 'allocID': req['allocID'], 'status': [s.to_dict() for s in status], 'proof': proof.to_dict(), }