source: fedd/federation/access.py @ a20a20f

#!/usr/local/bin/python

import os,sys
import stat # for chmod constants
import re
import random
import string
import copy
import pickle
import logging
import subprocess

from threading import *
from M2Crypto.SSL import SSLError

from util import *
from allocate_project import allocate_project_local, allocate_project_remote
from access_project import access_project
from fedid import fedid, generate_fedid
from authorizer import authorizer
from service_error import service_error
from remote_service import xmlrpc_handler, soap_handler, service_caller

import httplib
import tempfile
from urlparse import urlparse

import topdl
import list_log
import proxy_emulab_segment
import local_emulab_segment


# Make log messages disappear if noone configures a fedd logger
class nullHandler(logging.Handler):
    def emit(self, record): pass

fl = logging.getLogger("fedd.access")
fl.addHandler(nullHandler())

class access_base:
    """
    The implementation of access control based on mapping users to projects.

    Users can be mapped to existing projects or have projects created
    dynamically.  This implements both direct requests and proxies.
    """

    class parse_error(RuntimeError): pass

    def __init__(self, config=None, auth=None):
        """
        Initializer.  Pulls parameters out of the ConfigParser's access section.
        """

        # Make sure that the configuration is in place
        if not config: 
            raise RunTimeError("No config to fedd.access")

        self.project_priority = config.getboolean("access", "project_priority")

        self.certdir = config.get("access","certdir")
        self.create_debug = config.getboolean("access", "create_debug")
        self.cleanup = not config.getboolean("access", "leave_tmpfiles")
        self.access_type = config.get("access", "type")
        self.log = logging.getLogger("fedd.access")
        set_log_level(config, "access", self.log)
        self.state_lock = Lock()
        self.state = { }
        # XXX: Configurable
        self.exports = set(('SMB', 'seer', 'tmcd', 'userconfig', 
            'project_export', 'local_seer_control', 'seer_master', 
            'hide_hosts'))
        self.imports = set(('SMB', 'seer', 'userconfig', 'seer_master',
            'hide_hosts'))

        if auth: self.auth = auth
        else:
            self.log.error(\
                    "[access]: No authorizer initialized, creating local one.")
            auth = authorizer()

        self.state_filename = config.get("access", "access_state")
        self.read_state()

        # Keep cert_file and cert_pwd coming from the same place
        self.cert_file = config.get("access", "cert_file")
        if self.cert_file:
            self.sert_pwd = config.get("access", "cert_pw")
        else:
            self.cert_file = config.get("globals", "cert_file")
            self.sert_pwd = config.get("globals", "cert_pw")

        self.trusted_certs = config.get("access", "trusted_certs") or \
                config.get("globals", "trusted_certs")


    @staticmethod
    def software_list(v):
        """
        From a string containing a sequence of space separated pairs, return a
        list of tuples with pairs of location and file.
        """
        l = [ ]
        if v:
            ps = v.split(" ")
            while len(ps):
                loc, file = ps[0:2]
                del ps[0:2]
                l.append((loc, file))
        return l

    @staticmethod
    def add_kit(e, kit):
        """
        Add a Software object created from the list of (install, location)
        tuples passed as kit  to the software attribute of an object e.  We
        do this enough to break out the code, but it's kind of a hack to
        avoid changing the old tuple rep.
        """

        s = [ topdl.Software(install=i, location=l) for i, l in kit]

        if isinstance(e.software, list): e.software.extend(s)
        else: e.software = s 


    def read_access(self, config, access_obj=None):
        """
        Read an access DB with filename config  of the form:
            (id, id, id) -> attribute, something
        where the ids can be fedids, strings, or <any> or <none>, attribute is
        the attribute to assign , and something is any set of charcters.  The
        hash self.access is populated with mappings from those triples to the
        results of access_obj being called on the remainder of the line (if
        present).  If access_obj is not given, the string itself is entered in
        the hash.  Additionally, a triple with <any> and <none> mapped to None
        is entered in self.auth with the attribute given.

        Parsing errors result in a self.parse_error exception being raised.
        access_obj should throw that as well.
        """
        lineno=0
        name_expr = "["+string.ascii_letters + string.digits + "\.\-_]+"
        fedid_expr = "fedid:[" + string.hexdigits + "]+"
        key_name = "(<ANY>|<NONE>|"+fedid_expr + "|"+ name_expr + ")"
        access_re = re.compile('\('+key_name+'\s*,\s*'+key_name+'\s*,\s*'+
                key_name+'\s*\)\s*->\s*([^,]+)\s*(.*)', re.IGNORECASE)

        def parse_name(n):
            if n.startswith('fedid:'): return fedid(hexstr=n[len('fedid:'):])
            else: return n
        
        def auth_name(n):
            if isinstance(n, basestring):
                if n =='<any>' or n =='<none>': return None
                else: return unicode(n)
            else:
                return n
        def strip_comma(s):
            s = s.strip()
            if s.startswith(','):
                s = s[1:].strip()
            return s

        if access_obj is None:
            access_obj = lambda(x): "%s" % x

        f = open(config, "r");
        try:
            for line in f:
                lineno += 1
                line = line.strip();
                if len(line) == 0 or line.startswith('#'):
                    continue

                # Access line (t, p, u) -> anything
                m = access_re.match(line)
                if m != None:
                    access_key = tuple([ parse_name(x) \
                            for x in m.group(1,2,3)])
                    attribute = m.group(4)
                    auth_key = tuple([ auth_name(x) for x in access_key])
                    self.auth.set_attribute(auth_key, attribute)
                    if len(m.group(5)) > 0:
                        access_val = access_obj(strip_comma(m.group(5)))
                        self.access[access_key] = access_val
                    continue

                # Nothing matched to here: unknown line - raise exception
                f.close()
                raise self.parse_error(
                        "Unknown statement at line %d of %s" % \
                        (lineno, config))
        finally:
            if f: f.close()

    def get_users(self, obj):
        """
        Return a list of the IDs of the users in dict
        """
        if obj.has_key('user'):
            return [ unpack_id(u['userID']) \
                    for u in obj['user'] if u.has_key('userID') ]
        else:
            return None

    def write_state(self):
        if self.state_filename:
            try:
                f = open(self.state_filename, 'w')
                pickle.dump(self.state, f)
                self.log.debug("Wrote state to %s" % self.state_filename)
            except EnvironmentError, e:
                self.log.error("Can't write file %s: %s" % \
                        (self.state_filename, e))
            except pickle.PicklingError, e:
                self.log.error("Pickling problem: %s" % e)
            except TypeError, e:
                self.log.error("Pickling problem (TypeError): %s" % e)


    def read_state(self):
        """
        Read a new copy of access state.  Old state is overwritten.

        State format is a simple pickling of the state dictionary.
        """
        if self.state_filename:
            try:
                f = open(self.state_filename, "r")
                self.state = pickle.load(f)
                self.log.debug("[read_state]: Read state from %s" % \
                        self.state_filename)
            except EnvironmentError, e:
                self.log.warning(("[read_state]: No saved state: " +\
                        "Can't open %s: %s") % (self.state_filename, e))
            except EOFError, e:
                self.log.warning(("[read_state]: " +\
                        "Empty or damaged state file: %s:") % \
                        self.state_filename)
            except pickle.UnpicklingError, e:
                self.log.warning(("[read_state]: No saved state: " + \
                        "Unpickling failed: %s") % e)



    def permute_wildcards(self, a, p):
        """Return a copy of a with various fields wildcarded.

        The bits of p control the wildcards.  A set bit is a wildcard
        replacement with the lowest bit being user then project then testbed.
        """
        if p & 1: user = ["<any>"]
        else: user = a[2]
        if p & 2: proj = "<any>"
        else: proj = a[1]
        if p & 4: tb = "<any>"
        else: tb = a[0]

        return (tb, proj, user)

    def find_access(self, search):
        """
        Search the access DB for a match on this tuple.  Return the matching
        access tuple and the user that matched.
        
        NB, if the initial tuple fails to match we start inserting wildcards in
        an order determined by self.project_priority.  Try the list of users in
        order (when wildcarded, there's only one user in the list).
        """
        if self.project_priority: perm = (0, 1, 2, 3, 4, 5, 6, 7)
        else: perm = (0, 2, 1, 3, 4, 6, 5, 7)

        for p in perm: 
            s = self.permute_wildcards(search, p)
            # s[2] is None on an anonymous, unwildcarded request
            if s[2] != None:
                for u in s[2]:
                    if self.access.has_key((s[0], s[1], u)):
                        return (self.access[(s[0], s[1], u)], u)
            else:
                if self.access.has_key(s):
                    return (self.access[s], None)
        return None, None

    def lookup_access_base(self, req, fid):
        """
        Determine the allowed access for this request.  Return the access and
        which fields are dynamic.

        The fedid is needed to construct the request
        """
        user_re = re.compile("user:\s(.*)")
        project_re = re.compile("project:\s(.*)")

        # Search keys
        tb = fid
        user = [ user_re.findall(x)[0] for x in req.get('credential', []) \
                if user_re.match(x)]
        project = [ project_re.findall(x)[0] \
                for x in req.get('credential', []) \
                    if project_re.match(x)]

        if len(project) == 1: project = project[0]
        elif len(project) == 0: project = None
        else: 
            raise service_error(service_error.req, 
                    "More than one project credential")

        # Confirm authorization
        for u in user:
            self.log.debug("[lookup_access] Checking access for %s" % \
                    ((tb, project, u),))
            if self.auth.check_attribute((tb, project, u), 'access'):
                self.log.debug("[lookup_access] Access granted")
                break
            else:
                self.log.debug("[lookup_access] Access Denied")
        else:
            raise service_error(service_error.access, "Access denied")

        # This maps a valid user to the Emulab projects and users to use
        found, user_match = self.find_access((tb, project, user))

        return (found, (tb, project, user_match))
        

    def get_handler(self, path, fid):
        self.log.info("Get handler %s %s" % (path, fid))
        if self.auth.check_attribute(fid, path) and self.userconfdir:
            return ("%s/%s" % (self.userconfdir, path), "application/binary")
        else:
            return (None, None)

    def export_userconf(self, project):
        dev_null = None
        confid, confcert = generate_fedid("test", dir=self.userconfdir, 
                log=self.log)
        conffilename = "%s/%s" % (self.userconfdir, str(confid))
        cf = None
        try:
            cf = open(conffilename, "w")
            os.chmod(conffilename, stat.S_IRUSR | stat.S_IWUSR)
        except EnvironmentError, e:
            raise service_error(service_error.internal, 
                    "Cannot create user configuration data")

        try:
            dev_null = open("/dev/null", "a")
        except EnvironmentError, e:
            self.log.error("export_userconf: can't open /dev/null: %s" % e)

        cmd = "%s %s" % (self.userconfcmd, project)
        conf = subprocess.call(cmd.split(" "),
                stdout=cf, stderr=dev_null, close_fds=True)

        self.auth.set_attribute(confid, "/%s" % str(confid))

        return confid, confcert

    def export_SMB(self, id, state, project, user, attrs):
        return { 
                'id': id,
                'name': 'SMB',
                'visibility': 'export',
                'server': 'http://fs:139',
                'fedAttr': [
                        { 'attribute': 'SMBSHARE', 'value': 'USERS' },
                        { 'attribute': 'SMBUSER', 'value': user },
                        { 'attribute': 'SMBPROJ', 'value': project },
                    ]
                }

    def export_seer(self, id, state, project, user, attrs):
        return { 
                'id': id,
                'name': 'seer',
                'visibility': 'export',
                'server': 'http://control:16606',
                }

    def export_local_seer(self, id, state, project, user, attrs):
        return { 
                'id': id,
                'name': 'local_seer_control',
                'visibility': 'export',
                'server': 'http://control:16606',
                }

    def export_seer_master(self, id, state, project, user, attrs):
        return { 
                'id': id,
                'name': 'seer_master',
                'visibility': 'export',
                'server': 'http://seer-master:17707',
                }

    def export_tmcd(self, id, state, project, user, attrs):
        return { 
                'id': id,
                'name': 'seer',
                'visibility': 'export',
                'server': 'http://boss:7777',
                }

    def export_userconfig(self, id, state, project, user, attrs):
        if self.userconfdir and self.userconfcmd \
                and self.userconfurl:
            cid, cert = self.export_userconf(project)
            state['userconfig'] = unicode(cid)
            return {
                    'id': id,
                    'name': 'userconfig',
                    'visibility': 'export',
                    'server': "%s/%s" % (self.userconfurl, str(cid)),
                    'fedAttr': [
                        { 'attribute': 'cert', 'value': cert },
                    ]
                    }
        else:
            return None

    def export_hide_hosts(self, id, state, project, user, attrs):
        return {
                'id': id, 
                'name': 'hide_hosts',
                'visibility': 'export',
                'fedAttr': [ x for x in attrs \
                        if x.get('attribute', "") == 'hosts'],
                }

    def export_services(self, sreq, project, user):
        exp = [ ]
        state = { }
        # XXX: Filthy shortcut here using http: so urlparse will give the right
        # answers.
        for s in sreq:
            sname = s.get('name', '')
            svis = s.get('visibility', '')
            sattrs = s.get('fedAttr', [])
            if svis == 'export':
                if sname in self.exports:
                    id = s.get('id', 'no_id')
                    if sname == 'SMB':
                        exp.append(self.export_SMB(id, state, project, user,
                            sattrs))
                    elif sname == 'seer':
                        exp.append(self.export_seer(id, state, project, user,
                            sattrs))
                    elif sname == 'tmcd':
                        exp.append(self.export_tmcd(id, state, project, user,
                            sattrs))
                    elif sname == 'userconfig':
                        exp.append(self.export_userconfig(id, state,
                            project, user, sattrs))
                    elif sname == 'project_export':
                        exp.append(self.export_SMB(id, state, project, user,
                            sattrs))
                        #exp.append(self.export_seer(id, state, project, user,
                        #sattrs))
                        exp.append(self.export_userconfig(id, state,
                            project, user, sattrs))
                    elif sname == 'local_seer_control':
                        exp.append(self.export_local_seer(id, state, project,
                            user, sattrs))
                    elif sname == 'seer_master':
                        exp.append(self.export_seer_master(id, state, project,
                            user, sattrs))
                    elif sname == 'hide_hosts':
                        exp.append(self.export_hide_hosts(id, state, project,
                            user, sattrs))
        return (exp, state)

    def build_access_response(self, alloc_id, ap, services):
        """
        Create the SOAP response.

        Build the dictionary description of the response and use
        fedd_utils.pack_soap to create the soap message.  ap is the allocate
        project message returned from a remote project allocation (even if that
        allocation was done locally).
        """
        # Because alloc_id is already a fedd_services_types.IDType_Holder,
        # there's no need to repack it
        msg = { 
                'allocID': alloc_id,
                'fedAttr': [
                    { 'attribute': 'domain', 'value': self.domain } , 
                    { 'attribute': 'project', 'value': 
                        ap['project'].get('name', {}).get('localname', "???") },
                ]
            }

        if self.dragon_endpoint:
            msg['fedAttr'].append({'attribute': 'dragon',
                'value': self.dragon_endpoint})
        if self.deter_internal:
            print 'adding internal'
            msg['fedAttr'].append({'attribute': 'deter_internal',
                'value': self.deter_internal})
        else: print "internal: %s" % self.deter_internal
        #XXX: ??
        if self.dragon_vlans:
            msg['fedAttr'].append({'attribute': 'vlans',
                'value': self.dragon_vlans})

        if services:
            msg['service'] = services
        return msg

    def generate_portal_configs(self, topo, pubkey_base, secretkey_base, 
            tmpdir, lproj, leid, connInfo, services):

        def conninfo_to_dict(key, info):
            """
            Make a cpoy of the connection information about key, and flatten it
            into a single dict by parsing out any feddAttrs.
            """

            rv = None
            for i in info:
                if key == i.get('portal', "") or \
                        key in [e.get('element', "") \
                        for e in i.get('member', [])]:
                    rv = i.copy()
                    break

            else:
                return rv

            if 'fedAttr' in rv:
                for a in rv['fedAttr']:
                    attr = a.get('attribute', "")
                    val = a.get('value', "")
                    if attr and attr not in rv:
                        rv[attr] = val
                del rv['fedAttr']
            return rv

        # XXX: un hardcode this
        def client_null(f, s):
            print >>f, "Service: %s" % s['name']

        def client_seer_master(f, s):
            print >>f, 'PortalAlias: seer-master';

        def client_smb(f, s):
            print >>f, "Service: %s" % s['name']
            smbshare = None
            smbuser = None
            smbproj = None
            for a in s.get('fedAttr', []):
                if a.get('attribute', '') == 'SMBSHARE':
                    smbshare = a.get('value', None)
                elif a.get('attribute', '') == 'SMBUSER':
                    smbuser = a.get('value', None)
                elif a.get('attribute', '') == 'SMBPROJ':
                    smbproj = a.get('value', None)

            if all((smbshare, smbuser, smbproj)):
                print >>f, "SMBshare: %s" % smbshare
                print >>f, "ProjectUser: %s" % smbuser
                print >>f, "ProjectName: %s" % smbproj

        def client_hide_hosts(f, s):
            for a in s.get('fedAttr', [ ]):
                if a.get('attribute', "") == 'hosts':
                    print >>f, "Hide: %s" % a.get('value', "")

        client_service_out = {
                'SMB': client_smb,
                'tmcd': client_null,
                'seer': client_null,
                'userconfig': client_null,
                'project_export': client_null,
                'seer_master': client_seer_master,
                'hide_hosts': client_hide_hosts,
            }

        def client_seer_master_export(f, s):
            print >>f, "AddedNode: seer-master"

        def client_seer_local_export(f, s):
            print >>f, "AddedNode: control"

        client_export_service_out = {
                'seer_master': client_seer_master_export,
                'local_seer_control': client_seer_local_export,
            }

        def server_port(f, s):
            p = urlparse(s.get('server', 'http://localhost'))
            print >>f, 'port: remote:%s:%s:%s' % (p.port, p.hostname, p.port) 

        def server_null(f,s): pass

        def server_seer(f, s):
            print >>f, 'seer: True'

        server_service_out = {
                'SMB': server_port,
                'tmcd': server_port,
                'userconfig': server_null,
                'project_export': server_null,
                'seer': server_seer,
                'seer_master': server_port,
                'hide_hosts': server_null,
            }
        # XXX: end un hardcode this


        seer_out = False
        client_out = False
        mproj = None
        mexp = None
        control_gw = None
        testbed = ""
        # Create configuration files for the portals
        for e in [ e for e in topo.elements \
                if isinstance(e, topdl.Computer) and e.get_attribute('portal')]:
            myname = e.name
            type = e.get_attribute('portal_type')

            info = conninfo_to_dict(myname, connInfo)

            if not info:
                raise service_error(service_error.req,
                        "No connectivity info for %s" % myname)

            peer = info.get('peer', "")
            ldomain = self.domain;
            ssh_port = info.get('ssh_port', 22)

            # Collect this for the client.conf file
            if 'masterexperiment' in info:
                mproj, meid = info['masterexperiment'].split("/", 1)

            if type in ('control', 'both'):
                testbed = e.get_attribute('testbed')
                control_gw = myname

            active = info.get('active', 'False')

            cfn = "%s/%s.gw.conf" % (tmpdir, myname.lower())
            tunnelconfig = self.tunnel_config
            try:
                f = open(cfn, "w")
                if active == 'True':
                    print >>f, "active: True"
                    print >>f, "ssh_port: %s" % ssh_port
                    if type in ('control', 'both'):
                        for s in [s for s in services \
                                if s.get('name', "") in self.imports]:
                            server_service_out[s['name']](f, s)

                if tunnelconfig:
                    print >>f, "tunnelip: %s" % tunnelconfig
                print >>f, "peer: %s" % peer.lower()
                print >>f, "ssh_pubkey: /proj/%s/exp/%s/tmp/%s" % \
                        (lproj, leid, pubkey_base)
                print >>f, "ssh_privkey: /proj/%s/exp/%s/tmp/%s" % \
                        (lproj, leid, secretkey_base)
                f.close()
            except EnvironmentError, e:
                raise service_error(service_error.internal,
                        "Can't write protal config %s: %s" % (cfn, e))

        # Done with portals, write the client config file.
        try:
            f = open("%s/client.conf" % tmpdir, "w")
            if control_gw:
                print >>f, "ControlGateway: %s.%s.%s%s" % \
                    (myname.lower(), leid.lower(), lproj.lower(),
                            ldomain.lower())
            for s in services:
                if s.get('name',"") in self.imports and \
                        s.get('visibility','') == 'import':
                    client_service_out[s['name']](f, s)
                if s.get('name', '') in self.exports and \
                        s.get('visibility', '') == 'export' and \
                        s['name'] in client_export_service_out:
                    client_export_service_out[s['name']](f, s)
            # Seer uses this.
            if mproj and meid:
                print >>f, "ExperimentID: %s/%s" % (mproj, meid)
            f.close()
        except EnvironmentError, e:
            raise service_error(service_error.internal,
                    "Cannot write client.conf: %s" %s)

    def import_store_info(self, cf, connInfo):
        """
        Pull any import parameters in connInfo in.  We translate them either
        into known member names or fedAddrs.
        """

        for c in connInfo:
            for p in [ p for p in c.get('parameter', []) \
                    if p.get('type', '') == 'input']:
                name = p.get('name', None)
                key = p.get('key', None)
                store = p.get('store', None)

                if name and key and store :
                    req = { 'name': key, 'wait': True }
                    self.log.debug("Waiting for %s (%s) from %s" % \
                            (name, key, store))
                    r = self.call_GetValue(store, req, cf)
                    r = r.get('GetValueResponseBody', None)
                    if r :
                        if r.get('name', '') == key:
                            v = r.get('value', None)
                            if v is not None:
                                if name == 'peer':
                                    self.log.debug("Got peer %s" % v)
                                    c['peer'] = v
                                else:
                                    self.log.debug("Got %s %s" % (name, v))
                                    if c.has_key('fedAttr'):
                                        c['fedAttr'].append({
                                            'attribute': name, 'value': v})
                                    else:
                                        c['fedAttr']= [{
                                            'attribute': name, 'value': v}]
                            else:
                                raise service_error(service_error.internal, 
                                        'None value exported for %s'  % key)
                        else:
                            raise service_error(service_error.internal, 
                                    'Different name returned for %s: %s' \
                                            % (key, r.get('name','')))
                    else:
                        raise service_error(service_error.internal, 
                            'Badly formatted response: no GetValueResponseBody')
                else:
                    raise service_error(service_error.internal, 
                        'Bad Services missing info for import %s' % c)