#!/usr/local/bin/python import os,sys import stat # for chmod constants import re import random import string import copy import pickle import logging import subprocess import shlex from threading import * from M2Crypto.SSL import SSLError from util import * from deter import fedid, generate_fedid from authorizer import authorizer from service_error import service_error from remote_service import xmlrpc_handler, soap_handler, service_caller import httplib import tempfile from urlparse import urlparse from deter import topdl import list_log # Make log messages disappear if noone configures a fedd logger class nullHandler(logging.Handler): def emit(self, record): pass fl = logging.getLogger("fedd.access") fl.addHandler(nullHandler()) class access_base: """ The implementation of access control based on mapping users to projects. Users can be mapped to existing projects or have projects created dynamically. This implements both direct requests and proxies. """ class parse_error(RuntimeError): pass class access_attribute: def __init__(self, attr, value, pri=1): self.attr = attr self.value = value self.priority = pri def __str__(self): return "%s: %s (%d)" % (self.attr, self.value, self.priority) def __init__(self, config=None, auth=None): """ Initializer. Pulls parameters out of the ConfigParser's access section. """ # Make sure that the configuration is in place if not config: raise RunTimeError("No config to fedd.access") self.project_priority = config.getboolean("access", "project_priority") self.certdir = config.get("access","certdir") self.create_debug = config.getboolean("access", "create_debug") self.cleanup = not config.getboolean("access", "leave_tmpfiles") self.access_type = config.get("access", "type") self.log = logging.getLogger("fedd.access") set_log_level(config, "access", self.log) self.state_lock = Lock() self.state = { } # subclasses fill with what and how they export. self.exports = { } # XXX: Configurable self.imports = set(('SMB', 'seer', 'userconfig', 'seer_master', 'hide_hosts')) if auth: self.auth = auth else: self.log.error(\ "[access]: No authorizer initialized, creating local one.") auth = authorizer() self.state_filename = config.get("access", "access_state") self.read_state() # Keep cert_file and cert_pwd coming from the same place self.cert_file = config.get("access", "cert_file") if self.cert_file: self.cert_pwd = config.get("access", "cert_pw") else: self.cert_file = config.get("globals", "cert_file") self.sert_pwd = config.get("globals", "cert_pw") self.nat_portal = None self.trusted_certs = config.get("access", "trusted_certs") or \ config.get("globals", "trusted_certs") @staticmethod def software_list(v): """ From a string containing a sequence of space separated pairs, return a list of tuples with pairs of location and file. """ l = [ ] if v: ps = v.split(" ") while len(ps): loc, file = ps[0:2] del ps[0:2] l.append((loc, file)) return l @staticmethod def add_kit(e, kit): """ Add a Software object created from the list of (install, location) tuples passed as kit to the software attribute of an object e. We do this enough to break out the code, but it's kind of a hack to avoid changing the old tuple rep. """ s = [ topdl.Software(install=i, location=l) for i, l in kit] if isinstance(e.software, list): e.software.extend(s) else: e.software = s def read_access(self, fn, access_obj=None, default=[]): """ Read an access DB of the form abac.attribute -> local_auth_data The access dict is filled with mappings from the abac attributes (as strings) to the access objects. The objects are strings by default, but the class constructor is called with the string following the -> and whitespace in the file. """ map_re = re.compile("(\S+)\s+->\s+(.*)") priority_re = re.compile("([^,]+),\s*(\d+)") if access_obj is None: access_obj = lambda(x): "%s" % x self.access = [] priorities = { } f = open(fn, 'r') try: lineno = 0 for line in f: lineno += 1 line = line.strip(); if len(line) == 0 or line.startswith('#'): continue m = map_re.match(line) if m != None: self.access.append(access_base.access_attribute(m.group(1), access_obj(m.group(2)))) continue # If a priority is found, collect them m = priority_re.match(line) if m: try: priorities[m.group(1)] = int(m.group(2)) except ValueError, e: if self.log: self.log.debug("Bad priority in %s line %d" % \ (fn, lineno)) continue # Nothing matched to here: unknown line - raise exception # (finally will close f) raise self.parse_error( "Unknown statement at line %d of %s" % \ (lineno, fn)) finally: if f: f.close() # Set priorities for a in self.access: if a.attr in priorities: a.priority = priorities[a.attr] # default access mappings for a, v in default: self.access.append( access_base.access_attribute(attr=a, value=v, pri=0)) def write_state(self): if self.state_filename: try: f = open(self.state_filename, 'w') pickle.dump(self.state, f) self.log.debug("Wrote state to %s" % self.state_filename) except EnvironmentError, e: self.log.error("Can't write file %s: %s" % \ (self.state_filename, e)) except pickle.PicklingError, e: self.log.error("Pickling problem: %s" % e) except TypeError, e: self.log.error("Pickling problem (TypeError): %s" % e) def read_state(self): """ Read a new copy of access state. Old state is overwritten. State format is a simple pickling of the state dictionary. """ if self.state_filename: try: f = open(self.state_filename, "r") self.state = pickle.load(f) self.log.debug("[read_state]: Read state from %s" % \ self.state_filename) except EnvironmentError, e: self.log.warning(("[read_state]: No saved state: " +\ "Can't open %s: %s") % (self.state_filename, e)) except EOFError, e: self.log.warning(("[read_state]: " +\ "Empty or damaged state file: %s:") % \ self.state_filename) except pickle.UnpicklingError, e: self.log.warning(("[read_state]: No saved state: " + \ "Unpickling failed: %s") % e) def append_allocation_authorization(self, aid, attrs, need_state_lock=False, write_state_file=False, state_attr='state'): """ Append the authorization information to system state. By default we assume this is called with the state lock and with a write of the state file in the near future, need_state_lock and write_state_file can override this. The state_attr is the attribute in the access class that holds the per allocation information. Some complex classes use different names for the dict. """ for p, a in attrs: self.auth.set_attribute(p, a) self.auth.save() if need_state_lock: self.state_lock.acquire() d = getattr(self, state_attr) if aid in d and 'auth' in d[aid]: d[aid]['auth'].update(attrs) if write_state_file: self.write_state() if need_state_lock: self.state_lock.release() def clear_allocation_authorization(self, aid, need_state_lock=False, write_state_file=False, state_attr='state'): """ Attrs is a set of attribute principal pairs that need to be removed from the authenticator. Remove them and save the authenticator. See append_allocation_authorization for the various overrides. """ if need_state_lock: self.state_lock.acquire() d = getattr(self, state_attr) if aid in d and 'auth' in d[aid]: for p, a in d[aid]['auth']: self.auth.unset_attribute(p, a) d[aid]['auth'] = set() if write_state_file: self.write_state() if need_state_lock: self.state_lock.release() self.auth.save() def lookup_access(self, req, fid, filter=None, compare=None): """ Check all the attributes that this controller knows how to map and see if the requester is allowed to use any of them. If so return one. Filter defined the objects to check - it's a function that returns true for the objects to check - and cmp defines the order to check them in as the cmp field of sorted(). If filter is None, all possibilities are checked. If cmp is None, the choices are sorted by priority. """ # Import request credentials into this (clone later??) if self.auth.import_credentials( data_list=req.get('abac_credential', [])): self.auth.save() # NB: in the default case (the else), the comparison order is reversed # so numerically larger priorities are checked first. if compare: c = compare else: c = lambda a, b: cmp(b,a) if filter: f = filter else: f = lambda(x): True check = sorted([ a for a in self.access if f(a)], cmp=c) # Check every attribute that we know how to map and take the first # success. fail_proofs = [ ] for attr in check: access_ok, proof = self.auth.check_attribute(fid, attr.attr, with_proof=True) if access_ok: self.log.debug("Access succeeded for %s %s" % (attr.attr, fid)) return copy.copy(attr.value), [ fid ], proof else: fail_proofs.append(proof) self.log.debug("Access failed for %s %s" % (attr.attr, fid)) else: self.log.debug("Access denied for for %s" % fid) # We only return one fail proof because returning hundreds (which # is easy to do) locks up the fault mechanism. if len(fail_proofs) == 0: self.log.debug('No choices either: %s' % check) raise service_error(service_error.access, "Access denied - no way to grant requested access") raise service_error(service_error.access, "Access denied", proof=fail_proofs[0]) def get_handler(self, path, fid): """ This function is somewhat oddly named. It doesn't get a handler, it handles GETs. Specifically, it handls https GETs for retrieving data from the repository exported by the access server. """ self.log.info("Get handler %s %s" % (path, fid)) if len("%s" % fid) == 0: return (None, None) if self.auth.check_attribute(fid, path) and self.userconfdir: return ("%s/%s" % (self.userconfdir, path), "application/binary") else: return (None, None) def export_userconf(self, project): dev_null = None confid, confcert = generate_fedid("test", dir=self.userconfdir, log=self.log) conffilename = "%s/%s" % (self.userconfdir, str(confid)) cf = None try: cf = open(conffilename, "w") os.chmod(conffilename, stat.S_IRUSR | stat.S_IWUSR) except EnvironmentError, e: raise service_error(service_error.internal, "Cannot create user configuration data") try: dev_null = open("/dev/null", "a") except EnvironmentError, e: self.log.error("export_userconf: can't open /dev/null: %s" % e) cmd = "%s %s" % (self.userconfcmd, project) try: conf = subprocess.call(shlex.split(cmd), stdout=cf, stderr=dev_null, close_fds=True) except EnvironmentError,e: raise service_error(service_error.internal, "Could not run userconf command: %s %s" \ % (e.filename, e.strerror)) self.auth.set_attribute(confid, "/%s" % str(confid)) return confid, confcert def export_SMB(self, id, state, project, user, attrs): if project and user: return [{ 'id': id, 'name': 'SMB', 'visibility': 'export', 'server': 'http://fs:139', 'fedAttr': [ { 'attribute': 'SMBSHARE', 'value': 'USERS' }, { 'attribute': 'SMBUSER', 'value': user }, { 'attribute': 'SMBPROJ', 'value': project }, ] }] else: self.log.warn("Cannot export SMB w/o user and project") return [ ] def export_seer(self, id, state, project, user, attrs): return [{ 'id': id, 'name': 'seer', 'visibility': 'export', 'server': 'http://control:16606', }] def export_local_seer(self, id, state, project, user, attrs): return [{ 'id': id, 'name': 'local_seer_control', 'visibility': 'export', 'server': 'http://control:16606', }] def export_seer_master(self, id, state, project, user, attrs): return [{ 'id': id, 'name': 'seer_master', 'visibility': 'export', 'server': 'http://seer-master:17707', }] def export_tmcd(self, id, state, project, user, attrs): return [{ 'id': id, 'name': 'seer', 'visibility': 'export', 'server': 'http://boss:7777', }] def export_userconfig(self, id, state, project, user, attrs): if self.userconfdir and self.userconfcmd \ and self.userconfurl: cid, cert = self.export_userconf(project) state['userconfig'] = unicode(cid) return [{ 'id': id, 'name': 'userconfig', 'visibility': 'export', 'server': "%s/%s" % (self.userconfurl, str(cid)), 'fedAttr': [ { 'attribute': 'cert', 'value': cert }, ] }] else: return [ ] def export_hide_hosts(self, id, state, project, user, attrs): return [{ 'id': id, 'name': 'hide_hosts', 'visibility': 'export', 'fedAttr': [ x for x in attrs \ if x.get('attribute', "") == 'hosts'], }] def export_project_export(self, id, state, project, user, attrs): rv = [ ] rv.extend(self.export_SMB(id, state, project, user, attrs)) rv.extend(self.export_userconfig(id, state, project, user, attrs)) return rv def export_services(self, sreq, project=None, user=None): exp = [ ] state = { } for s in sreq: sname = s.get('name', '') svis = s.get('visibility', '') sattrs = s.get('fedAttr', []) if svis == 'export': if sname in self.exports: id = s.get('id', 'no_id') exp.extend(self.exports[sname](id, state, project, user, sattrs)) return (exp, state) def build_access_response(self, alloc_id, pname, services, proof): """ Create the SOAP response. Build the dictionary description of the response and use fedd_utils.pack_soap to create the soap message. ap is the allocate project message returned from a remote project allocation (even if that allocation was done locally). """ # Because alloc_id is already a fedd_services_types.IDType_Holder, # there's no need to repack it msg = { 'allocID': alloc_id, 'proof': proof.to_dict(), 'fedAttr': [ { 'attribute': 'domain', 'value': self.domain } , ] } if pname: msg['fedAttr'].append({ 'attribute': 'project', 'value': pname }) if self.dragon_endpoint: msg['fedAttr'].append({'attribute': 'dragon', 'value': self.dragon_endpoint}) if self.deter_internal: msg['fedAttr'].append({'attribute': 'deter_internal', 'value': self.deter_internal}) #XXX: ?? if self.dragon_vlans: msg['fedAttr'].append({'attribute': 'vlans', 'value': self.dragon_vlans}) if self.nat_portal is not None: msg['fedAttr'].append({'attribute': 'nat_portals', 'value': True}) if services: msg['service'] = services return msg def generate_portal_configs(self, topo, pubkey_base, secretkey_base, tmpdir, lproj, leid, connInfo, services): def conninfo_to_dict(key, info): """ Make a cpoy of the connection information about key, and flatten it into a single dict by parsing out any feddAttrs. """ rv = None for i in info: if key == i.get('portal', "") or \ key in [e.get('element', "") \ for e in i.get('member', [])]: rv = i.copy() break else: return rv if 'fedAttr' in rv: for a in rv['fedAttr']: attr = a.get('attribute', "") val = a.get('value', "") if attr and attr not in rv: rv[attr] = val del rv['fedAttr'] return rv # XXX: un hardcode this def client_null(f, s): print >>f, "Service: %s" % s['name'] def client_seer_master(f, s): print >>f, 'PortalAlias: seer-master' def client_smb(f, s): print >>f, "Service: %s" % s['name'] smbshare = None smbuser = None smbproj = None for a in s.get('fedAttr', []): if a.get('attribute', '') == 'SMBSHARE': smbshare = a.get('value', None) elif a.get('attribute', '') == 'SMBUSER': smbuser = a.get('value', None) elif a.get('attribute', '') == 'SMBPROJ': smbproj = a.get('value', None) if all((smbshare, smbuser, smbproj)): print >>f, "SMBshare: %s" % smbshare print >>f, "ProjectUser: %s" % smbuser print >>f, "ProjectName: %s" % smbproj def client_hide_hosts(f, s): for a in s.get('fedAttr', [ ]): if a.get('attribute', "") == 'hosts': print >>f, "Hide: %s" % a.get('value', "") client_service_out = { 'SMB': client_smb, 'tmcd': client_null, 'seer': client_null, 'userconfig': client_null, 'project_export': client_null, 'seer_master': client_seer_master, 'hide_hosts': client_hide_hosts, } def client_seer_master_export(f, s): print >>f, "AddedNode: seer-master" def client_seer_local_export(f, s): print >>f, "AddedNode: control" client_export_service_out = { 'seer_master': client_seer_master_export, 'local_seer_control': client_seer_local_export, } def server_port(f, s): p = urlparse(s.get('server', 'http://localhost')) print >>f, 'port: remote:%s:%s:%s' % (p.port, p.hostname, p.port) def server_null(f,s): pass def server_seer(f, s): print >>f, 'seer: True' server_service_out = { 'SMB': server_port, 'tmcd': server_port, 'userconfig': server_null, 'project_export': server_null, 'seer': server_seer, 'seer_master': server_port, 'hide_hosts': server_null, } # XXX: end un hardcode this seer_out = False client_out = False mproj = None mexp = None control_gw = None testbed = "" # Create configuration files for the portals for e in [ e for e in topo.elements \ if isinstance(e, topdl.Computer) and e.get_attribute('portal')]: myname = e.name type = e.get_attribute('portal_type') info = conninfo_to_dict(myname, connInfo) if not info: raise service_error(service_error.req, "No connectivity info for %s" % myname) peer = info.get('peer', "") ldomain = self.domain ssh_port = info.get('ssh_port', 22) # Collect this for the client.conf file if 'masterexperiment' in info: mproj, meid = info['masterexperiment'].split("/", 1) if type in ('control', 'both'): testbed = e.get_attribute('testbed') control_gw = myname active = info.get('active', 'False') nat_partner = info.get('nat_partner', 'False') cfn = "%s/%s.gw.conf" % (tmpdir, myname.lower()) tunnelconfig = self.tunnel_config try: f = open(cfn, "w") if active == 'True': print >>f, "active: True" print >>f, "ssh_port: %s" % ssh_port if type in ('control', 'both'): for s in [s for s in services \ if s.get('name', "") in self.imports]: server_service_out[s['name']](f, s) if nat_partner == 'True': print >>f, "nat_partner: True" if tunnelconfig: print >>f, "tunnelip: %s" % tunnelconfig print >>f, "peer: %s" % peer.lower() print >>f, "ssh_pubkey: /proj/%s/exp/%s/tmp/%s" % \ (lproj, leid, pubkey_base) print >>f, "ssh_privkey: /proj/%s/exp/%s/tmp/%s" % \ (lproj, leid, secretkey_base) f.close() except EnvironmentError, e: raise service_error(service_error.internal, "Can't write protal config %s: %s" % (cfn, e)) # Done with portals, write the client config file. try: f = open("%s/client.conf" % tmpdir, "w") if control_gw: print >>f, "ControlGateway: %s.%s.%s%s" % \ (myname.lower(), leid.lower(), lproj.lower(), ldomain.lower()) for s in services: if s.get('name',"") in self.imports and \ s.get('visibility','') == 'import': client_service_out[s['name']](f, s) if s.get('name', '') in self.exports and \ s.get('visibility', '') == 'export' and \ s['name'] in client_export_service_out: client_export_service_out[s['name']](f, s) # Seer uses this. if mproj and meid: print >>f, "ExperimentID: %s/%s" % (mproj, meid) f.close() except EnvironmentError, e: raise service_error(service_error.internal, "Cannot write client.conf: %s" %s) def configure_userconf(self, services, tmpdir): """ If the userconf service was imported, collect the configuration data. """ for s in services: s_name = s.get('name', '') s_vis = s.get('visibility','') if s_name == 'userconfig' and s_vis == 'import': # Collect ther server and certificate info. u = s.get('server', None) for a in s.get('fedAttr', []): if a.get('attribute',"") == 'cert': cert = a.get('value', None) break else: cert = None if cert: # Make a temporary certificate file for get_url. The # finally clause removes it whether something goes # wrong (including an exception from get_url) or not. try: tfos, tn = tempfile.mkstemp(suffix=".pem") tf = os.fdopen(tfos, 'w') print >>tf, cert tf.close() self.log.debug("Getting userconf info: %s" % u) get_url(u, tn, tmpdir, "userconf") self.log.debug("Got userconf info: %s" % u) except EnvironmentError, e: raise service_error(service.error.internal, "Cannot create temp file for " + "userconfig certificates: %s" % e) except: t, v, st = sys.exc_info() raise service_error(service_error.internal, "Error retrieving %s: %s" % (u, v)) finally: if tn: os.remove(tn) else: raise service_error(service_error.req, "No certificate for retreiving userconfig") break def import_store_info(self, cf, connInfo): """ Pull any import parameters in connInfo in. We translate them either into known member names or fedAddrs. """ for c in connInfo: for p in [ p for p in c.get('parameter', []) \ if p.get('type', '') == 'input']: name = p.get('name', None) key = p.get('key', None) store = p.get('store', None) if not all((name, key, store)): raise service_error(service_error.internal, 'Bad Services missing info for import %s' % c) req = { 'name': key, 'wait': True } self.log.debug("Waiting for %s (%s) from %s" % \ (name, key, store)) r = self.call_GetValue(store, req, cf) r = r.get('GetValueResponseBody', None) if r is None: raise service_error(service_error.internal, 'Badly formatted response: no GetValueResponseBody') if r.get('name', '') != key: raise service_error(service_error.internal, 'Different name returned for %s: %s' \ % (key, r.get('name',''))) v = r.get('value', None) if v is None: raise service_error(service_error.internal, 'None value exported for %s' % key) if name == 'peer': self.log.debug("Got peer %s" % v) c['peer'] = v else: self.log.debug("Got %s %s" % (name, v)) if c.has_key('fedAttr'): c['fedAttr'].append({ 'attribute': name, 'value': v}) else: c['fedAttr']= [{ 'attribute': name, 'value': v}] def remove_dirs(self, dir): """ Remove the directory tree and all files rooted at dir. Log any errors, but continue. """ self.log.debug("[removedirs]: removing %s" % dir) try: for path, dirs, files in os.walk(dir, topdown=False): for f in files: os.remove(os.path.join(path, f)) for d in dirs: os.rmdir(os.path.join(path, d)) os.rmdir(dir) except EnvironmentError, e: self.log.error("Error deleting directory tree in %s" % e); def RequestAccess(self, req, fid): """ Handle an access request. Success here maps the requester into the local access control space and establishes state about that user keyed to a fedid. We also save a copy of the certificate underlying that fedid so this allocation can access configuration information and shared parameters on the experiment controller. """ self.log.info("RequestAccess called by %s" % fid) # The dance to get into the request body if req.has_key('RequestAccessRequestBody'): req = req['RequestAccessRequestBody'] else: raise service_error(service_error.req, "No request!?") # Base class lookup routine. If this fails, it throws a service # exception denying access that triggers a fault response back to the # caller. found, owners, proof = self.lookup_access(req, fid) self.log.info( "[RequestAccess] Access granted local creds %s" % found) # Make a fedid for this allocation allocID, alloc_cert = generate_fedid(subj="alloc", log=self.log) aid = unicode(allocID) # Store the data about this allocation: self.state_lock.acquire() self.state[aid] = { } self.state[aid]['user'] = found self.state[aid]['owners'] = owners self.state[aid]['auth'] = set() # Authorize the creating fedid and the principal representing the # allocation to manipulate it. self.append_allocation_authorization(aid, ((fid, allocID), (allocID, allocID))) self.write_state() self.state_lock.release() # Create a directory to stash the certificate in, ans stash it. try: f = open("%s/%s.pem" % (self.certdir, aid), "w") print >>f, alloc_cert f.close() except EnvironmentError, e: raise service_error(service_error.internal, "Can't open %s/%s : %s" % (self.certdir, aid, e)) self.log.debug('[RequestAccess] Returning allocation ID: %s' % allocID) return { 'allocID': { 'fedid': allocID }, 'proof': proof.to_dict() } def ReleaseAccess(self, req, fid): """ Release the allocation granted earlier. Access to the allocation is checked and if valid, the state and cached certificate are destroyed. """ self.log.info("ReleaseAccess called by %s" % fid) # The dance to get into the request body if req.has_key('ReleaseAccessRequestBody'): req = req['ReleaseAccessRequestBody'] else: raise service_error(service_error.req, "No request!?") # Pull a key out of the request. One can request to delete an # allocation by a local human readable name or by a fedid. This finds # both choices. try: if 'localname' in req['allocID']: auth_attr = aid = req['allocID']['localname'] elif 'fedid' in req['allocID']: aid = unicode(req['allocID']['fedid']) auth_attr = req['allocID']['fedid'] else: raise service_error(service_error.req, "Only localnames and fedids are understood") except KeyError: raise service_error(service_error.req, "Badly formed request") self.log.debug("[ReleaseAccess] deallocation requested for %s", aid) # Confirm access access_ok, proof = self.auth.check_attribute(fid, auth_attr, with_proof=True) if not access_ok: self.log.debug("[ReleaseAccess] deallocation denied for %s", aid) raise service_error(service_error.access, "Access Denied", proof=proof) # If there is an allocation in the state, delete it. Note the locking. self.state_lock.acquire() if aid in self.state: self.log.debug("[ReleaseAccess] Found allocation for %s" %aid) self.clear_allocation_authorization(aid) del self.state[aid] self.write_state() self.state_lock.release() # And remove the access cert cf = "%s/%s.pem" % (self.certdir, aid) self.log.debug("[ReleaseAccess] Removing %s" % cf) os.remove(cf) self.log.info("ReleaseAccess succeeded for %s" % fid) return { 'allocID': req['allocID'], 'proof': proof.to_dict() } else: self.state_lock.release() raise service_error(service_error.req, "No such allocation")