source: fedd/federation/access.py @ cd60510

#!/usr/local/bin/python

import os,sys
import stat # for chmod constants
import re
import random
import string
import copy
import pickle
import logging
import subprocess

from threading import *
from M2Crypto.SSL import SSLError

from util import *
from allocate_project import allocate_project_local, allocate_project_remote
from fedid import fedid, generate_fedid
from authorizer import authorizer
from service_error import service_error
from remote_service import xmlrpc_handler, soap_handler, service_caller

import httplib
import tempfile
from urlparse import urlparse

import topdl
import list_log
import proxy_emulab_segment
import local_emulab_segment


# Make log messages disappear if noone configures a fedd logger
class nullHandler(logging.Handler):
    def emit(self, record): pass

fl = logging.getLogger("fedd.access")
fl.addHandler(nullHandler())

class access_base:
    """
    The implementation of access control based on mapping users to projects.

    Users can be mapped to existing projects or have projects created
    dynamically.  This implements both direct requests and proxies.
    """

    class parse_error(RuntimeError): pass

    class access_attribute:
        def __init__(self, attr, value, pri=1):
            self.attr = attr
            self.value = value
            self.priority = pri
        def __str__(self):
            return "%s: %s (%d)" % (self.attr, self.value, self.priority)

    def __init__(self, config=None, auth=None):
        """
        Initializer.  Pulls parameters out of the ConfigParser's access section.
        """

        # Make sure that the configuration is in place
        if not config: 
            raise RunTimeError("No config to fedd.access")

        self.project_priority = config.getboolean("access", "project_priority")

        self.certdir = config.get("access","certdir")
        self.create_debug = config.getboolean("access", "create_debug")
        self.cleanup = not config.getboolean("access", "leave_tmpfiles")
        self.access_type = config.get("access", "type")
        self.log = logging.getLogger("fedd.access")
        set_log_level(config, "access", self.log)
        self.state_lock = Lock()
        self.state = { }
        # subclasses fill with what and how they export.
        self.exports = { }
        # XXX: Configurable
        self.imports = set(('SMB', 'seer', 'userconfig', 'seer_master',
            'hide_hosts'))

        if auth: self.auth = auth
        else:
            self.log.error(\
                    "[access]: No authorizer initialized, creating local one.")
            auth = authorizer()

        self.state_filename = config.get("access", "access_state")
        self.read_state()

        # Keep cert_file and cert_pwd coming from the same place
        self.cert_file = config.get("access", "cert_file")
        if self.cert_file:
            self.cert_pwd = config.get("access", "cert_pw")
        else:
            self.cert_file = config.get("globals", "cert_file")
            self.sert_pwd = config.get("globals", "cert_pw")

        self.trusted_certs = config.get("access", "trusted_certs") or \
                config.get("globals", "trusted_certs")


    @staticmethod
    def software_list(v):
        """
        From a string containing a sequence of space separated pairs, return a
        list of tuples with pairs of location and file.
        """
        l = [ ]
        if v:
            ps = v.split(" ")
            while len(ps):
                loc, file = ps[0:2]
                del ps[0:2]
                l.append((loc, file))
        return l

    @staticmethod
    def add_kit(e, kit):
        """
        Add a Software object created from the list of (install, location)
        tuples passed as kit  to the software attribute of an object e.  We
        do this enough to break out the code, but it's kind of a hack to
        avoid changing the old tuple rep.
        """

        s = [ topdl.Software(install=i, location=l) for i, l in kit]

        if isinstance(e.software, list): e.software.extend(s)
        else: e.software = s 


    def read_access(self, fn, access_obj=None, default=[]):
        """
        Read an access DB of the form
            abac.attribute -> local_auth_data
        The access dict is filled with mappings from the abac attributes (as
        strings) to the access objects.  The objects are strings by default,
        but the class constructor is called with the string following the ->
        and whitespace in the file.
        """

        map_re = re.compile("(\S+)\s+->\s+(.*)")
        priority_re = re.compile("([^,]+),\s*(\d+)")

        if access_obj is None:
            access_obj = lambda(x): "%s" % x

        self.access = []
        priorities = { }

        f = open(fn, 'r')
        try:
            lineno = 0
            for line in f:
                lineno += 1
                line = line.strip();
                if len(line) == 0 or line.startswith('#'):
                    continue
                m = map_re.match(line)
                if m != None:
                    self.access.append(access_base.access_attribute(m.group(1),
                        access_obj(m.group(2))))
                    continue

                # If a priority is found, collect them
                m = priority_re.match(line)
                if m:
                    try:
                        priorities[m.group(1)] = int(m.group(2))
                    except ValueError, e:
                        if self.log:
                            self.log.debug("Bad priority in %s line %d" % \
                                    (fn, lineno))
                    continue

                # Nothing matched to here: unknown line - raise exception
                # (finally will close f)
                raise self.parse_error(
                        "Unknown statement at line %d of %s" % \
                        (lineno, fn))
        finally:
            if f: f.close()

        # Set priorities
        for a in self.access:
            if a.attr in priorities:
                a.priority = priorities[a.attr]

        # default access mappings
        for a, v in default:
            self.access.append(
                    access_base.access_attribute(attr=a, value=v, pri=0))



    def write_state(self):
        if self.state_filename:
            try:
                f = open(self.state_filename, 'w')
                pickle.dump(self.state, f)
                self.log.debug("Wrote state to %s" % self.state_filename)
            except EnvironmentError, e:
                self.log.error("Can't write file %s: %s" % \
                        (self.state_filename, e))
            except pickle.PicklingError, e:
                self.log.error("Pickling problem: %s" % e)
            except TypeError, e:
                self.log.error("Pickling problem (TypeError): %s" % e)


    def read_state(self):
        """
        Read a new copy of access state.  Old state is overwritten.

        State format is a simple pickling of the state dictionary.
        """
        if self.state_filename:
            try:
                f = open(self.state_filename, "r")
                self.state = pickle.load(f)
                self.log.debug("[read_state]: Read state from %s" % \
                        self.state_filename)
            except EnvironmentError, e:
                self.log.warning(("[read_state]: No saved state: " +\
                        "Can't open %s: %s") % (self.state_filename, e))
            except EOFError, e:
                self.log.warning(("[read_state]: " +\
                        "Empty or damaged state file: %s:") % \
                        self.state_filename)
            except pickle.UnpicklingError, e:
                self.log.warning(("[read_state]: No saved state: " + \
                        "Unpickling failed: %s") % e)

    def lookup_access(self, req, fid, filter=None, compare=None): 
        """
        Check all the attributes that this controller knows how to map and see
        if the requester is allowed to use any of them.  If so return one.
        Filter defined the objects to check - it's a function that returns true
        for the objects to check - and cmp defines the order to check them in
        as the cmp field of sorted().  If filter is None, all possibilities are
        checked.  If cmp is None, the choices are sorted by priority.
        """

        # Import request credentials into this (clone later??)
        if self.auth.import_credentials(
                data_list=req.get('abac_credential', [])):
            self.auth.save()

        # NB: in the default case (the else), the comparison order is reversed
        # so numerically larger priorities are checked first.
        if compare: c = compare
        else: c = lambda a, b: cmp(b,a)

        if filter: f = filter
        else: f = lambda(x): True

        check = sorted([ a for a in self.access if f(a)], cmp=c)

        # Check every attribute that we know how to map and take the first
        # success.
        for attr in check:
            if self.auth.check_attribute(fid, attr.attr):
                self.log.debug("Access succeeded for %s %s" % (attr.attr, fid))
                # XXX: needs to deal with dynamics
                return copy.copy(attr.value), (False, False, False), \
                        [ fid ]
            else:
                self.log.debug("Access failed for %s %s" % (attr.attr, fid))
        else:
            raise service_error(service_error.access, "Access denied")



    def get_handler(self, path, fid):
        """
        This function is somewhat oddly named.  It doesn't get a handler, it
        handles GETs.  Specifically, it handls https GETs for retrieving data
        from the repository exported by the access server.
        """
        self.log.info("Get handler %s %s" % (path, fid))
        if self.auth.check_attribute(fid, path) and self.userconfdir:
            return ("%s/%s" % (self.userconfdir, path), "application/binary")
        else:
            return (None, None)

    def export_userconf(self, project):
        dev_null = None
        confid, confcert = generate_fedid("test", dir=self.userconfdir, 
                log=self.log)
        conffilename = "%s/%s" % (self.userconfdir, str(confid))
        cf = None
        try:
            cf = open(conffilename, "w")
            os.chmod(conffilename, stat.S_IRUSR | stat.S_IWUSR)
        except EnvironmentError, e:
            raise service_error(service_error.internal, 
                    "Cannot create user configuration data")

        try:
            dev_null = open("/dev/null", "a")
        except EnvironmentError, e:
            self.log.error("export_userconf: can't open /dev/null: %s" % e)

        cmd = "%s %s" % (self.userconfcmd, project)
        conf = subprocess.call(cmd.split(" "),
                stdout=cf, stderr=dev_null, close_fds=True)

        self.auth.set_attribute(confid, "/%s" % str(confid))

        return confid, confcert

    def export_SMB(self, id, state, project, user, attrs):
        if project and user:
            return [{ 
                    'id': id,
                    'name': 'SMB',
                    'visibility': 'export',
                    'server': 'http://fs:139',
                    'fedAttr': [
                            { 'attribute': 'SMBSHARE', 'value': 'USERS' },
                            { 'attribute': 'SMBUSER', 'value': user },
                            { 'attribute': 'SMBPROJ', 'value': project },
                        ]
                    }]
        else:
            self.log.warn("Cannot export SMB w/o user and project")
            return [ ]

    def export_seer(self, id, state, project, user, attrs):
        return [{ 
                'id': id,
                'name': 'seer',
                'visibility': 'export',
                'server': 'http://control:16606',
                }]

    def export_local_seer(self, id, state, project, user, attrs):
        return [{ 
                'id': id,
                'name': 'local_seer_control',
                'visibility': 'export',
                'server': 'http://control:16606',
                }]

    def export_seer_master(self, id, state, project, user, attrs):
        return [{ 
                'id': id,
                'name': 'seer_master',
                'visibility': 'export',
                'server': 'http://seer-master:17707',
                }]

    def export_tmcd(self, id, state, project, user, attrs):
        return [{ 
                'id': id,
                'name': 'seer',
                'visibility': 'export',
                'server': 'http://boss:7777',
                }]

    def export_userconfig(self, id, state, project, user, attrs):
        if self.userconfdir and self.userconfcmd \
                and self.userconfurl:
            cid, cert = self.export_userconf(project)
            state['userconfig'] = unicode(cid)
            return [{
                    'id': id,
                    'name': 'userconfig',
                    'visibility': 'export',
                    'server': "%s/%s" % (self.userconfurl, str(cid)),
                    'fedAttr': [
                        { 'attribute': 'cert', 'value': cert },
                    ]
                    }]
        else:
            return [ ]

    def export_hide_hosts(self, id, state, project, user, attrs):
        return [{
                'id': id, 
                'name': 'hide_hosts',
                'visibility': 'export',
                'fedAttr': [ x for x in attrs \
                        if x.get('attribute', "") == 'hosts'],
                }]

    def export_project_export(self, id, state, project, user, attrs):
        rv = [ ]
        rv.extend(self.export_SMB(id, state, project, user, attrs))
        rv.extend(self.export_userconfig(id, state, project, user, attrs))
        return rv

    def export_services(self, sreq, project=None, user=None):
        exp = [ ]
        state = { }
        for s in sreq:
            sname = s.get('name', '')
            svis = s.get('visibility', '')
            sattrs = s.get('fedAttr', [])
            if svis == 'export':
                if sname in self.exports:
                    id = s.get('id', 'no_id')
                    exp.extend(self.exports[sname](id, state, project, user,
                            sattrs))

        return (exp, state)

    def build_access_response(self, alloc_id, ap, services):
        """
        Create the SOAP response.

        Build the dictionary description of the response and use
        fedd_utils.pack_soap to create the soap message.  ap is the allocate
        project message returned from a remote project allocation (even if that
        allocation was done locally).
        """
        # Because alloc_id is already a fedd_services_types.IDType_Holder,
        # there's no need to repack it
        msg = { 
                'allocID': alloc_id,
                'fedAttr': [
                    { 'attribute': 'domain', 'value': self.domain } , 
                    { 'attribute': 'project', 'value': 
                        ap['project'].get('name', {}).get('localname', "???") },
                ]
            }

        if self.dragon_endpoint:
            msg['fedAttr'].append({'attribute': 'dragon',
                'value': self.dragon_endpoint})
        if self.deter_internal:
            msg['fedAttr'].append({'attribute': 'deter_internal',
                'value': self.deter_internal})
        #XXX: ??
        if self.dragon_vlans:
            msg['fedAttr'].append({'attribute': 'vlans',
                'value': self.dragon_vlans})

        if services:
            msg['service'] = services
        return msg

    def generate_portal_configs(self, topo, pubkey_base, secretkey_base, 
            tmpdir, lproj, leid, connInfo, services):

        def conninfo_to_dict(key, info):
            """
            Make a cpoy of the connection information about key, and flatten it
            into a single dict by parsing out any feddAttrs.
            """

            rv = None
            for i in info:
                if key == i.get('portal', "") or \
                        key in [e.get('element', "") \
                        for e in i.get('member', [])]:
                    rv = i.copy()
                    break

            else:
                return rv

            if 'fedAttr' in rv:
                for a in rv['fedAttr']:
                    attr = a.get('attribute', "")
                    val = a.get('value', "")
                    if attr and attr not in rv:
                        rv[attr] = val
                del rv['fedAttr']
            return rv

        # XXX: un hardcode this
        def client_null(f, s):
            print >>f, "Service: %s" % s['name']

        def client_seer_master(f, s):
            print >>f, 'PortalAlias: seer-master'

        def client_smb(f, s):
            print >>f, "Service: %s" % s['name']
            smbshare = None
            smbuser = None
            smbproj = None
            for a in s.get('fedAttr', []):
                if a.get('attribute', '') == 'SMBSHARE':
                    smbshare = a.get('value', None)
                elif a.get('attribute', '') == 'SMBUSER':
                    smbuser = a.get('value', None)
                elif a.get('attribute', '') == 'SMBPROJ':
                    smbproj = a.get('value', None)

            if all((smbshare, smbuser, smbproj)):
                print >>f, "SMBshare: %s" % smbshare
                print >>f, "ProjectUser: %s" % smbuser
                print >>f, "ProjectName: %s" % smbproj

        def client_hide_hosts(f, s):
            for a in s.get('fedAttr', [ ]):
                if a.get('attribute', "") == 'hosts':
                    print >>f, "Hide: %s" % a.get('value', "")

        client_service_out = {
                'SMB': client_smb,
                'tmcd': client_null,
                'seer': client_null,
                'userconfig': client_null,
                'project_export': client_null,
                'seer_master': client_seer_master,
                'hide_hosts': client_hide_hosts,
            }

        def client_seer_master_export(f, s):
            print >>f, "AddedNode: seer-master"

        def client_seer_local_export(f, s):
            print >>f, "AddedNode: control"

        client_export_service_out = {
                'seer_master': client_seer_master_export,
                'local_seer_control': client_seer_local_export,
            }

        def server_port(f, s):
            p = urlparse(s.get('server', 'http://localhost'))
            print >>f, 'port: remote:%s:%s:%s' % (p.port, p.hostname, p.port) 

        def server_null(f,s): pass

        def server_seer(f, s):
            print >>f, 'seer: True'

        server_service_out = {
                'SMB': server_port,
                'tmcd': server_port,
                'userconfig': server_null,
                'project_export': server_null,
                'seer': server_seer,
                'seer_master': server_port,
                'hide_hosts': server_null,
            }
        # XXX: end un hardcode this


        seer_out = False
        client_out = False
        mproj = None
        mexp = None
        control_gw = None
        testbed = ""
        # Create configuration files for the portals
        for e in [ e for e in topo.elements \
                if isinstance(e, topdl.Computer) and e.get_attribute('portal')]:
            myname = e.name
            type = e.get_attribute('portal_type')

            info = conninfo_to_dict(myname, connInfo)

            if not info:
                raise service_error(service_error.req,
                        "No connectivity info for %s" % myname)

            peer = info.get('peer', "")
            ldomain = self.domain
            ssh_port = info.get('ssh_port', 22)

            # Collect this for the client.conf file
            if 'masterexperiment' in info:
                mproj, meid = info['masterexperiment'].split("/", 1)

            if type in ('control', 'both'):
                testbed = e.get_attribute('testbed')
                control_gw = myname

            active = info.get('active', 'False')

            cfn = "%s/%s.gw.conf" % (tmpdir, myname.lower())
            tunnelconfig = self.tunnel_config
            try:
                f = open(cfn, "w")
                if active == 'True':
                    print >>f, "active: True"
                    print >>f, "ssh_port: %s" % ssh_port
                    if type in ('control', 'both'):
                        for s in [s for s in services \
                                if s.get('name', "") in self.imports]:
                            server_service_out[s['name']](f, s)

                if tunnelconfig:
                    print >>f, "tunnelip: %s" % tunnelconfig
                print >>f, "peer: %s" % peer.lower()
                print >>f, "ssh_pubkey: /proj/%s/exp/%s/tmp/%s" % \
                        (lproj, leid, pubkey_base)
                print >>f, "ssh_privkey: /proj/%s/exp/%s/tmp/%s" % \
                        (lproj, leid, secretkey_base)
                f.close()
            except EnvironmentError, e:
                raise service_error(service_error.internal,
                        "Can't write protal config %s: %s" % (cfn, e))

        # Done with portals, write the client config file.
        try:
            f = open("%s/client.conf" % tmpdir, "w")
            if control_gw:
                print >>f, "ControlGateway: %s.%s.%s%s" % \
                    (myname.lower(), leid.lower(), lproj.lower(),
                            ldomain.lower())
            for s in services:
                if s.get('name',"") in self.imports and \
                        s.get('visibility','') == 'import':
                    client_service_out[s['name']](f, s)
                if s.get('name', '') in self.exports and \
                        s.get('visibility', '') == 'export' and \
                        s['name'] in client_export_service_out:
                    client_export_service_out[s['name']](f, s)
            # Seer uses this.
            if mproj and meid:
                print >>f, "ExperimentID: %s/%s" % (mproj, meid)
            f.close()
        except EnvironmentError, e:
            raise service_error(service_error.internal,
                    "Cannot write client.conf: %s" %s)

    def configure_userconf(self, services, tmpdir):
        """
        If the userconf service was imported, collect the configuration data.
        """
        for s in services:
            s_name = s.get('name', '')
            s_vis = s.get('visibility','')
            if s_name  == 'userconfig' and s_vis == 'import':
                # Collect ther server and certificate info.
                u = s.get('server', None)
                for a in s.get('fedAttr', []):
                    if a.get('attribute',"") == 'cert':
                        cert = a.get('value', None)
                        break
                else:
                    cert = None

                if cert:
                    # Make a temporary certificate file for get_url.  The
                    # finally clause removes it whether something goes
                    # wrong (including an exception from get_url) or not.
                    try:
                        tfos, tn = tempfile.mkstemp(suffix=".pem")
                        tf = os.fdopen(tfos, 'w')
                        print >>tf, cert
                        tf.close()
                        self.log.debug("Getting userconf info: %s" % u)
                        get_url(u, tn, tmpdir, "userconf")
                        self.log.debug("Got userconf info: %s" % u)
                    except EnvironmentError, e:
                        raise service_error(service.error.internal, 
                                "Cannot create temp file for " + 
                                "userconfig certificates: %s" % e)
                    except:
                        t, v, st = sys.exc_info()
                        raise service_error(service_error.internal,
                                "Error retrieving %s: %s" % (u, v))
                    finally:
                        if tn: os.remove(tn)
                else:
                    raise service_error(service_error.req,
                            "No certificate for retreiving userconfig")
                break

    def import_store_info(self, cf, connInfo):
        """
        Pull any import parameters in connInfo in.  We translate them either
        into known member names or fedAddrs.
        """

        for c in connInfo:
            for p in [ p for p in c.get('parameter', []) \
                    if p.get('type', '') == 'input']:
                name = p.get('name', None)
                key = p.get('key', None)
                store = p.get('store', None)

                if name and key and store :
                    req = { 'name': key, 'wait': True }
                    self.log.debug("Waiting for %s (%s) from %s" % \
                            (name, key, store))
                    r = self.call_GetValue(store, req, cf)
                    r = r.get('GetValueResponseBody', None)
                    if r :
                        if r.get('name', '') == key:
                            v = r.get('value', None)
                            if v is not None:
                                if name == 'peer':
                                    self.log.debug("Got peer %s" % v)
                                    c['peer'] = v
                                else:
                                    self.log.debug("Got %s %s" % (name, v))
                                    if c.has_key('fedAttr'):
                                        c['fedAttr'].append({
                                            'attribute': name, 'value': v})
                                    else:
                                        c['fedAttr']= [{
                                            'attribute': name, 'value': v}]
                            else:
                                raise service_error(service_error.internal, 
                                        'None value exported for %s'  % key)
                        else:
                            raise service_error(service_error.internal, 
                                    'Different name returned for %s: %s' \
                                            % (key, r.get('name','')))
                    else:
                        raise service_error(service_error.internal, 
                            'Badly formatted response: no GetValueResponseBody')
                else:
                    raise service_error(service_error.internal, 
                        'Bad Services missing info for import %s' % c)

    def remove_dirs(self, dir):
        """
        Remove the directory tree and all files rooted at dir.  Log any errors,
        but continue.
        """
        self.log.debug("[removedirs]: removing %s" % dir)
        try:
            for path, dirs, files in os.walk(dir, topdown=False):
                for f in files:
                    os.remove(os.path.join(path, f))
                for d in dirs:
                    os.rmdir(os.path.join(path, d))
            os.rmdir(dir)
        except EnvironmentError, e:
            self.log.error("Error deleting directory tree in %s" % e);