#!/usr/local/bin/python import os,sys import stat # for chmod constants import re import random import string import copy import pickle import logging import subprocess from threading import * from M2Crypto.SSL import SSLError from util import * from allocate_project import allocate_project_local, allocate_project_remote from access_project import access_project from fedid import fedid, generate_fedid from authorizer import authorizer from service_error import service_error from remote_service import xmlrpc_handler, soap_handler, service_caller import httplib import tempfile from urlparse import urlparse import topdl import list_log import proxy_emulab_segment import local_emulab_segment # Make log messages disappear if noone configures a fedd logger class nullHandler(logging.Handler): def emit(self, record): pass fl = logging.getLogger("fedd.access") fl.addHandler(nullHandler()) class access_base: """ The implementation of access control based on mapping users to projects. Users can be mapped to existing projects or have projects created dynamically. This implements both direct requests and proxies. """ class parse_error(RuntimeError): pass def __init__(self, config=None, auth=None): """ Initializer. Pulls parameters out of the ConfigParser's access section. """ # Make sure that the configuration is in place if not config: raise RunTimeError("No config to fedd.access") self.project_priority = config.getboolean("access", "project_priority") self.certdir = config.get("access","certdir") self.create_debug = config.getboolean("access", "create_debug") self.cleanup = not config.getboolean("access", "leave_tmpfiles") self.access_type = config.get("access", "type") self.log = logging.getLogger("fedd.access") set_log_level(config, "access", self.log) self.state_lock = Lock() self.state = { } # subclasses fill with what and how they export. self.exports = { } # XXX: Configurable self.imports = set(('SMB', 'seer', 'userconfig', 'seer_master', 'hide_hosts')) if auth: self.auth = auth else: self.log.error(\ "[access]: No authorizer initialized, creating local one.") auth = authorizer() self.state_filename = config.get("access", "access_state") self.read_state() # Keep cert_file and cert_pwd coming from the same place self.cert_file = config.get("access", "cert_file") if self.cert_file: self.cert_pwd = config.get("access", "cert_pw") else: self.cert_file = config.get("globals", "cert_file") self.sert_pwd = config.get("globals", "cert_pw") self.trusted_certs = config.get("access", "trusted_certs") or \ config.get("globals", "trusted_certs") @staticmethod def software_list(v): """ From a string containing a sequence of space separated pairs, return a list of tuples with pairs of location and file. """ l = [ ] if v: ps = v.split(" ") while len(ps): loc, file = ps[0:2] del ps[0:2] l.append((loc, file)) return l @staticmethod def add_kit(e, kit): """ Add a Software object created from the list of (install, location) tuples passed as kit to the software attribute of an object e. We do this enough to break out the code, but it's kind of a hack to avoid changing the old tuple rep. """ s = [ topdl.Software(install=i, location=l) for i, l in kit] if isinstance(e.software, list): e.software.extend(s) else: e.software = s def read_access(self, config, access_obj=None): """ Read an access DB with filename config of the form: (id, id, id) -> attribute, something where the ids can be fedids, strings, or or , attribute is the attribute to assign , and something is any set of charcters. The hash self.access is populated with mappings from those triples to the results of access_obj being called on the remainder of the line (if present). If access_obj is not given, the string itself is entered in the hash. Additionally, a triple with and mapped to None is entered in self.auth with the attribute given. Parsing errors result in a self.parse_error exception being raised. access_obj should throw that as well. """ lineno=0 name_expr = "["+string.ascii_letters + string.digits + "\.\-_]+" fedid_expr = "fedid:[" + string.hexdigits + "]+" key_name = "(||"+fedid_expr + "|"+ name_expr + ")" access_re = re.compile('\('+key_name+'\s*,\s*'+key_name+'\s*,\s*'+ key_name+'\s*\)\s*->\s*([^,]+)\s*(.*)', re.IGNORECASE) def parse_name(n): if n.startswith('fedid:'): return fedid(hexstr=n[len('fedid:'):]) else: return n def auth_name(n): if isinstance(n, basestring): if n =='' or n =='': return None else: return unicode(n) else: return n def strip_comma(s): s = s.strip() if s.startswith(','): s = s[1:].strip() return s if access_obj is None: access_obj = lambda(x): "%s" % x f = open(config, "r"); try: for line in f: lineno += 1 line = line.strip(); if len(line) == 0 or line.startswith('#'): continue # Access line (t, p, u) -> anything m = access_re.match(line) if m != None: access_key = tuple([ parse_name(x) \ for x in m.group(1,2,3)]) attribute = m.group(4) auth_key = tuple([ auth_name(x) for x in access_key]) self.auth.set_attribute(auth_key, attribute) if len(m.group(5)) > 0: access_val = access_obj(strip_comma(m.group(5))) self.access[access_key] = access_val continue # Nothing matched to here: unknown line - raise exception f.close() raise self.parse_error( "Unknown statement at line %d of %s" % \ (lineno, config)) finally: if f: f.close() def get_users(self, obj): """ Return a list of the IDs of the users in dict """ if obj.has_key('user'): return [ unpack_id(u['userID']) \ for u in obj['user'] if u.has_key('userID') ] else: return None def write_state(self): if self.state_filename: try: f = open(self.state_filename, 'w') pickle.dump(self.state, f) self.log.debug("Wrote state to %s" % self.state_filename) except EnvironmentError, e: self.log.error("Can't write file %s: %s" % \ (self.state_filename, e)) except pickle.PicklingError, e: self.log.error("Pickling problem: %s" % e) except TypeError, e: self.log.error("Pickling problem (TypeError): %s" % e) def read_state(self): """ Read a new copy of access state. Old state is overwritten. State format is a simple pickling of the state dictionary. """ if self.state_filename: try: f = open(self.state_filename, "r") self.state = pickle.load(f) self.log.debug("[read_state]: Read state from %s" % \ self.state_filename) except EnvironmentError, e: self.log.warning(("[read_state]: No saved state: " +\ "Can't open %s: %s") % (self.state_filename, e)) except EOFError, e: self.log.warning(("[read_state]: " +\ "Empty or damaged state file: %s:") % \ self.state_filename) except pickle.UnpicklingError, e: self.log.warning(("[read_state]: No saved state: " + \ "Unpickling failed: %s") % e) def permute_wildcards(self, a, p): """Return a copy of a with various fields wildcarded. The bits of p control the wildcards. A set bit is a wildcard replacement with the lowest bit being user then project then testbed. """ if p & 1: user = [""] else: user = a[2] if p & 2: proj = "" else: proj = a[1] if p & 4: tb = "" else: tb = a[0] return (tb, proj, user) def find_access(self, search): """ Search the access DB for a match on this tuple. Return the matching access tuple and the user that matched. NB, if the initial tuple fails to match we start inserting wildcards in an order determined by self.project_priority. Try the list of users in order (when wildcarded, there's only one user in the list). """ if self.project_priority: perm = (0, 1, 2, 3, 4, 5, 6, 7) else: perm = (0, 2, 1, 3, 4, 6, 5, 7) for p in perm: s = self.permute_wildcards(search, p) # s[2] is None on an anonymous, unwildcarded request if s[2] != None: for u in s[2]: if self.access.has_key((s[0], s[1], u)): return (self.access[(s[0], s[1], u)], u) else: if self.access.has_key(s): return (self.access[s], None) return None, None def lookup_access_base(self, req, fid): """ Determine the allowed access for this request. Return the access and which fields are dynamic. The fedid is needed to construct the request """ user_re = re.compile("user:\s(.*)") project_re = re.compile("project:\s(.*)") # Search keys tb = fid user = [ user_re.findall(x)[0] for x in req.get('credential', []) \ if user_re.match(x)] project = [ project_re.findall(x)[0] \ for x in req.get('credential', []) \ if project_re.match(x)] if len(project) == 1: project = project[0] elif len(project) == 0: project = None else: raise service_error(service_error.req, "More than one project credential") # Confirm authorization for u in user: self.log.debug("[lookup_access] Checking access for %s" % \ ((tb, project, u),)) if self.auth.check_attribute((tb, project, u), 'access'): self.log.debug("[lookup_access] Access granted") break else: self.log.debug("[lookup_access] Access Denied") else: raise service_error(service_error.access, "Access denied") # This maps a valid user to the Emulab projects and users to use found, user_match = self.find_access((tb, project, user)) return (found, (tb, project, user_match)) def get_handler(self, path, fid): self.log.info("Get handler %s %s" % (path, fid)) if self.auth.check_attribute(fid, path) and self.userconfdir: return ("%s/%s" % (self.userconfdir, path), "application/binary") else: return (None, None) def export_userconf(self, project): dev_null = None confid, confcert = generate_fedid("test", dir=self.userconfdir, log=self.log) conffilename = "%s/%s" % (self.userconfdir, str(confid)) cf = None try: cf = open(conffilename, "w") os.chmod(conffilename, stat.S_IRUSR | stat.S_IWUSR) except EnvironmentError, e: raise service_error(service_error.internal, "Cannot create user configuration data") try: dev_null = open("/dev/null", "a") except EnvironmentError, e: self.log.error("export_userconf: can't open /dev/null: %s" % e) cmd = "%s %s" % (self.userconfcmd, project) conf = subprocess.call(cmd.split(" "), stdout=cf, stderr=dev_null, close_fds=True) self.auth.set_attribute(confid, "/%s" % str(confid)) return confid, confcert def export_SMB(self, id, state, project, user, attrs): if project and user: return [{ 'id': id, 'name': 'SMB', 'visibility': 'export', 'server': 'http://fs:139', 'fedAttr': [ { 'attribute': 'SMBSHARE', 'value': 'USERS' }, { 'attribute': 'SMBUSER', 'value': user }, { 'attribute': 'SMBPROJ', 'value': project }, ] }] else: self.log.warn("Cannot export SMB w/o user and project") return [ ] def export_seer(self, id, state, project, user, attrs): return [{ 'id': id, 'name': 'seer', 'visibility': 'export', 'server': 'http://control:16606', }] def export_local_seer(self, id, state, project, user, attrs): return [{ 'id': id, 'name': 'local_seer_control', 'visibility': 'export', 'server': 'http://control:16606', }] def export_seer_master(self, id, state, project, user, attrs): return [{ 'id': id, 'name': 'seer_master', 'visibility': 'export', 'server': 'http://seer-master:17707', }] def export_tmcd(self, id, state, project, user, attrs): return [{ 'id': id, 'name': 'seer', 'visibility': 'export', 'server': 'http://boss:7777', }] def export_userconfig(self, id, state, project, user, attrs): if self.userconfdir and self.userconfcmd \ and self.userconfurl: cid, cert = self.export_userconf(project) state['userconfig'] = unicode(cid) return [{ 'id': id, 'name': 'userconfig', 'visibility': 'export', 'server': "%s/%s" % (self.userconfurl, str(cid)), 'fedAttr': [ { 'attribute': 'cert', 'value': cert }, ] }] else: return [ ] def export_hide_hosts(self, id, state, project, user, attrs): return [{ 'id': id, 'name': 'hide_hosts', 'visibility': 'export', 'fedAttr': [ x for x in attrs \ if x.get('attribute', "") == 'hosts'], }] def export_project_export(self, id, state, project, user, attrs): rv = [ ] rv.extend(self.export_SMB(id, state, project, user, attrs)) rv.extend(self.export_userconfig(id, state, project, user, attrs)) return rv def export_services(self, sreq, project=None, user=None): exp = [ ] state = { } for s in sreq: sname = s.get('name', '') svis = s.get('visibility', '') sattrs = s.get('fedAttr', []) if svis == 'export': if sname in self.exports: id = s.get('id', 'no_id') exp.extend(self.exports[sname](id, state, project, user, sattrs)) return (exp, state) def build_access_response(self, alloc_id, ap, services): """ Create the SOAP response. Build the dictionary description of the response and use fedd_utils.pack_soap to create the soap message. ap is the allocate project message returned from a remote project allocation (even if that allocation was done locally). """ # Because alloc_id is already a fedd_services_types.IDType_Holder, # there's no need to repack it msg = { 'allocID': alloc_id, 'fedAttr': [ { 'attribute': 'domain', 'value': self.domain } , { 'attribute': 'project', 'value': ap['project'].get('name', {}).get('localname', "???") }, ] } if self.dragon_endpoint: msg['fedAttr'].append({'attribute': 'dragon', 'value': self.dragon_endpoint}) if self.deter_internal: msg['fedAttr'].append({'attribute': 'deter_internal', 'value': self.deter_internal}) #XXX: ?? if self.dragon_vlans: msg['fedAttr'].append({'attribute': 'vlans', 'value': self.dragon_vlans}) if services: msg['service'] = services return msg def generate_portal_configs(self, topo, pubkey_base, secretkey_base, tmpdir, lproj, leid, connInfo, services): def conninfo_to_dict(key, info): """ Make a cpoy of the connection information about key, and flatten it into a single dict by parsing out any feddAttrs. """ rv = None for i in info: if key == i.get('portal', "") or \ key in [e.get('element', "") \ for e in i.get('member', [])]: rv = i.copy() break else: return rv if 'fedAttr' in rv: for a in rv['fedAttr']: attr = a.get('attribute', "") val = a.get('value', "") if attr and attr not in rv: rv[attr] = val del rv['fedAttr'] return rv # XXX: un hardcode this def client_null(f, s): print >>f, "Service: %s" % s['name'] def client_seer_master(f, s): print >>f, 'PortalAlias: seer-master' def client_smb(f, s): print >>f, "Service: %s" % s['name'] smbshare = None smbuser = None smbproj = None for a in s.get('fedAttr', []): if a.get('attribute', '') == 'SMBSHARE': smbshare = a.get('value', None) elif a.get('attribute', '') == 'SMBUSER': smbuser = a.get('value', None) elif a.get('attribute', '') == 'SMBPROJ': smbproj = a.get('value', None) if all((smbshare, smbuser, smbproj)): print >>f, "SMBshare: %s" % smbshare print >>f, "ProjectUser: %s" % smbuser print >>f, "ProjectName: %s" % smbproj def client_hide_hosts(f, s): for a in s.get('fedAttr', [ ]): if a.get('attribute', "") == 'hosts': print >>f, "Hide: %s" % a.get('value', "") client_service_out = { 'SMB': client_smb, 'tmcd': client_null, 'seer': client_null, 'userconfig': client_null, 'project_export': client_null, 'seer_master': client_seer_master, 'hide_hosts': client_hide_hosts, } def client_seer_master_export(f, s): print >>f, "AddedNode: seer-master" def client_seer_local_export(f, s): print >>f, "AddedNode: control" client_export_service_out = { 'seer_master': client_seer_master_export, 'local_seer_control': client_seer_local_export, } def server_port(f, s): p = urlparse(s.get('server', 'http://localhost')) print >>f, 'port: remote:%s:%s:%s' % (p.port, p.hostname, p.port) def server_null(f,s): pass def server_seer(f, s): print >>f, 'seer: True' server_service_out = { 'SMB': server_port, 'tmcd': server_port, 'userconfig': server_null, 'project_export': server_null, 'seer': server_seer, 'seer_master': server_port, 'hide_hosts': server_null, } # XXX: end un hardcode this seer_out = False client_out = False mproj = None mexp = None control_gw = None testbed = "" # Create configuration files for the portals for e in [ e for e in topo.elements \ if isinstance(e, topdl.Computer) and e.get_attribute('portal')]: myname = e.name type = e.get_attribute('portal_type') info = conninfo_to_dict(myname, connInfo) if not info: raise service_error(service_error.req, "No connectivity info for %s" % myname) peer = info.get('peer', "") ldomain = self.domain ssh_port = info.get('ssh_port', 22) # Collect this for the client.conf file if 'masterexperiment' in info: mproj, meid = info['masterexperiment'].split("/", 1) if type in ('control', 'both'): testbed = e.get_attribute('testbed') control_gw = myname active = info.get('active', 'False') cfn = "%s/%s.gw.conf" % (tmpdir, myname.lower()) tunnelconfig = self.tunnel_config try: f = open(cfn, "w") if active == 'True': print >>f, "active: True" print >>f, "ssh_port: %s" % ssh_port if type in ('control', 'both'): for s in [s for s in services \ if s.get('name', "") in self.imports]: server_service_out[s['name']](f, s) if tunnelconfig: print >>f, "tunnelip: %s" % tunnelconfig print >>f, "peer: %s" % peer.lower() print >>f, "ssh_pubkey: /proj/%s/exp/%s/tmp/%s" % \ (lproj, leid, pubkey_base) print >>f, "ssh_privkey: /proj/%s/exp/%s/tmp/%s" % \ (lproj, leid, secretkey_base) f.close() except EnvironmentError, e: raise service_error(service_error.internal, "Can't write protal config %s: %s" % (cfn, e)) # Done with portals, write the client config file. try: f = open("%s/client.conf" % tmpdir, "w") if control_gw: print >>f, "ControlGateway: %s.%s.%s%s" % \ (myname.lower(), leid.lower(), lproj.lower(), ldomain.lower()) for s in services: if s.get('name',"") in self.imports and \ s.get('visibility','') == 'import': client_service_out[s['name']](f, s) if s.get('name', '') in self.exports and \ s.get('visibility', '') == 'export' and \ s['name'] in client_export_service_out: client_export_service_out[s['name']](f, s) # Seer uses this. if mproj and meid: print >>f, "ExperimentID: %s/%s" % (mproj, meid) f.close() except EnvironmentError, e: raise service_error(service_error.internal, "Cannot write client.conf: %s" %s) def configure_userconf(self, services, tmpdir): """ If the userconf service was imported, collect the configuration data. """ for s in services: s_name = s.get('name', '') s_vis = s.get('visibility','') if s_name == 'userconfig' and s_vis == 'import': # Collect ther server and certificate info. u = s.get('server', None) for a in s.get('fedAttr', []): if a.get('attribute',"") == 'cert': cert = a.get('value', None) break else: cert = None if cert: # Make a temporary certificate file for get_url. The # finally clause removes it whether something goes # wrong (including an exception from get_url) or not. try: tfos, tn = tempfile.mkstemp(suffix=".pem") tf = os.fdopen(tfos, 'w') print >>tf, cert tf.close() self.log.debug("Getting userconf info: %s" % u) get_url(u, tn, tmpdir, "userconf") self.log.debug("Got userconf info: %s" % u) except EnvironmentError, e: raise service_error(service.error.internal, "Cannot create temp file for " + "userconfig certificates: %s" % e) except: t, v, st = sys.exc_info() raise service_error(service_error.internal, "Error retrieving %s: %s" % (u, v)) finally: if tn: os.remove(tn) else: raise service_error(service_error.req, "No certificate for retreiving userconfig") break def import_store_info(self, cf, connInfo): """ Pull any import parameters in connInfo in. We translate them either into known member names or fedAddrs. """ for c in connInfo: for p in [ p for p in c.get('parameter', []) \ if p.get('type', '') == 'input']: name = p.get('name', None) key = p.get('key', None) store = p.get('store', None) if name and key and store : req = { 'name': key, 'wait': True } self.log.debug("Waiting for %s (%s) from %s" % \ (name, key, store)) r = self.call_GetValue(store, req, cf) r = r.get('GetValueResponseBody', None) if r : if r.get('name', '') == key: v = r.get('value', None) if v is not None: if name == 'peer': self.log.debug("Got peer %s" % v) c['peer'] = v else: self.log.debug("Got %s %s" % (name, v)) if c.has_key('fedAttr'): c['fedAttr'].append({ 'attribute': name, 'value': v}) else: c['fedAttr']= [{ 'attribute': name, 'value': v}] else: raise service_error(service_error.internal, 'None value exported for %s' % key) else: raise service_error(service_error.internal, 'Different name returned for %s: %s' \ % (key, r.get('name',''))) else: raise service_error(service_error.internal, 'Badly formatted response: no GetValueResponseBody') else: raise service_error(service_error.internal, 'Bad Services missing info for import %s' % c) def remove_dirs(self, dir): """ Remove the directory tree and all files rooted at dir. Log any errors, but continue. """ self.log.debug("[removedirs]: removing %s" % dir) try: for path, dirs, files in os.walk(dir, topdown=False): for f in files: os.remove(os.path.join(path, f)) for d in dirs: os.rmdir(os.path.join(path, d)) os.rmdir(dir) except EnvironmentError, e: self.log.error("Error deleting directory tree in %s" % e);