source: fedd/federation/util.py @ dcac546

Last change on this file since dcac546 was 4241f3c, checked in by Ted Faber <faber@…>, 12 years ago

Allow protocol setting

  • Property mode set to 100644
File size: 12.5 KB
Line 
1#!/usr/local/bin/python
2
3import re
4import os
5import string
6import logging
7import pickle
8
9import httplib
10
11from optparse import OptionParser
12
13from socket import sslerror
14from tempfile import mkstemp
15
16from M2Crypto import SSL
17from M2Crypto.SSL import SSLError
18from deter import fedid
19from service_error import service_error
20from urlparse import urlparse
21from M2Crypto import m2
22
23
24# If this is an old enough version of M2Crypto.SSL that has an
25# ssl_verify_callback that doesn't allow 0-length signed certs, create a
26# version of that callback that does.  This is edited from the original in
27# M2Crypto.SSL.cb.  This version also elides the printing to stderr.
28if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None):
29    from M2Crypto.SSL.Context import map
30
31    def fedd_ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
32        unknown_issuer = [
33            m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY,
34            m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE,
35            m2.X509_V_ERR_CERT_UNTRUSTED,
36            m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
37            ]
38        # m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
39        if getattr(m2, 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None):
40            unknown_issuer.append(getattr(m2, 
41                'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None))
42        ssl_ctx = map()[ssl_ctx_ptr]
43
44        if errnum in unknown_issuer: 
45            if ssl_ctx.get_allow_unknown_ca():
46                ok = 1
47        # CRL checking goes here...
48        if ok:
49            if ssl_ctx.get_verify_depth() >= errdepth:
50                ok = 1
51            else:
52                ok = 0
53        return ok
54else:
55    def fedd_ssl_verify_callback(ok, store):
56        '''
57        m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
58        '''
59        errnum = store.get_error()
60        if errnum == m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
61            ok = 1
62            return ok
63        else:
64            return SSL.cb.ssl_verify_callback_allow_unknown_ca(ok, store)
65
66class fedd_ssl_context(SSL.Context):
67    """
68    Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd.
69    """
70    def __init__(self, my_cert, trusted_certs=None, password=None, protocol=None):
71        """
72        construct a fedd_ssl_context
73
74        @param my_cert: PEM file with my certificate in it
75        @param trusted_certs: PEM file with trusted certs in it (optional)
76        """
77        if protocol: SSL.Context.__init__(self, protocol=protocol)
78        else: SSL.Context.__init__(self)
79
80        # load_cert takes a callback to get a password, not a password, so if
81        # the caller provided a password, this creates a nonce callback using a
82        # lambda form.
83        if password != None and not callable(password):
84            # This is cute.  password = lambda *args: password produces a
85            # function object that returns itself rather than one that returns
86            # the object itself.  This is because password is an object
87            # reference and after the assignment it's a lambda.  So we assign
88            # to a temp.
89            pwd = str(password)
90            password =lambda *args: pwd
91
92        # The calls to str below (and above) are because the underlying SSL
93        # stuff is intolerant of unicode.
94        if password != None:
95            self.load_cert(str(my_cert), callback=password)
96        else:
97            self.load_cert(str(my_cert))
98
99        # If no trusted certificates are specified, allow unknown CAs.
100        if trusted_certs: 
101            self.load_verify_locations(trusted_certs)
102            self.set_verify(SSL.verify_peer, 10)
103        else:
104            # Install the proper callback to allow self-signed certs
105            self.set_allow_unknown_ca(True)
106            self.set_verify(SSL.verify_peer, 10, 
107                    callback=fedd_ssl_verify_callback)
108
109        # no session caching
110        self.set_session_cache_mode(0)
111
112class file_expanding_opts(OptionParser):
113    def expand_file(self, option, opt_str, v, p):
114        """
115        Store the given value to the given destination after expanding home
116        directories.
117        """
118        setattr(p.values, option.dest, os.path.expanduser(v))
119
120    def __init__(self, usage=None, version=None):
121        OptionParser.__init__(self)
122
123
124def read_simple_accessdb(fn, auth, mask=[]):
125    """
126    Read a simple access database.  Each line is a fedid (of the form
127    fedid:hexstring) and a comma separated list of atributes to be assigned to
128    it.  This parses out the fedids and adds the attributes to the authorizer.
129    comments (preceded with a #) and blank lines are ignored.  Exceptions (e.g.
130    file exceptions and ValueErrors from badly parsed lines) are propagated.
131    """
132
133    rv = [ ]
134    lineno = 0
135    fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\
136            "(\w+\s*(,\s*\w+)*)")
137
138    # If a single string came in, make it a list
139    if isinstance(mask, basestring): mask = [ mask ]
140
141    f = open(fn, 'r')
142    for line in f:
143        lineno += 1
144        line = line.strip()
145        if line.startswith('#') or len(line) == 0: 
146            continue
147        m = fedid_line.match(line)
148        if m :
149            fid = fedid(hexstr=m.group(1))
150            for a in [ a.strip() for a in m.group(2).split(",") \
151                    if not mask or a.strip() in mask ]:
152                auth.set_attribute(fid, a.strip())
153        else:
154            raise ValueError("Badly formatted line in accessdb: %s line %d" %\
155                    (fn, lineno))
156    f.close()
157    return rv
158       
159
160def pack_id(id):
161    """
162    Return a dictionary with the field name set by the id type.  Handy for
163    creating dictionaries to be converted to messages.
164    """
165    if isinstance(id, fedid): return { 'fedid': id }
166    elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id }
167    else: return { 'localname': id}
168
169def unpack_id(id):
170    """return id as a type determined by the key"""
171    for k in ("localname", "fedid", "uri", "kerberosUsername"):
172        if id.has_key(k): return id[k]
173    return None
174
175def set_log_level(config, sect, log):
176    """ Set the logging level to the value passed in sect of config."""
177    # The getattr sleight of hand finds the logging level constant
178    # corrersponding to the string.  We're a little paranoid to avoid user
179    # mayhem.
180    if config.has_option(sect, "log_level"):
181        level_str = config.get(sect, "log_level")
182        try:
183            level = int(getattr(logging, level_str.upper(), -1))
184
185            if  logging.DEBUG <= level <= logging.CRITICAL:
186                log.setLevel(level)
187            else:
188                log.error("Bad experiment_log value: %s" % level_str)
189
190        except ValueError:
191            log.error("Bad experiment_log value: %s" % level_str)
192
193def copy_file(src, dest, size=1024):
194    """
195    Exceedingly simple file copy.  Throws an EnvironmentError if there's a problem.
196    """
197    s = open(src,'r')
198    d = open(dest, 'w')
199
200    buf = s.read(size)
201    while buf != "":
202        d.write(buf)
203        buf = s.read(size)
204    s.close()
205    d.close()
206
207def get_url(url, cf, destdir, fn=None, max_retries=5, log=None):
208    """
209    Get data from a federated data store.  This presents the client cert/fedid
210    to the http server.  We retry up to max_retries times.
211    """
212    po = urlparse(url)
213    if not fn:
214        fn = po.path.rpartition('/')[2]
215    retries = 0
216    ok = False
217    failed_exception = None
218    while not ok and retries < max_retries:
219        try:
220            if log:
221                log.debug('Creating HTTPSConnection: %s' % cf)
222            conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
223                    cert_file=cf, key_file=cf, timeout=30)
224            conn.putrequest('GET', po.path)
225            conn.endheaders()
226            if log:
227                log.debug('Connecting')
228            response = conn.getresponse()
229            if log:
230                log.debug('Connected')
231
232            lf = open("%s/%s" % (destdir, fn), "w")
233            if log:
234                log.debug('initial read')
235            buf = response.read(4096)
236            while buf:
237                lf.write(buf)
238                if log:
239                    log.debug('loop read')
240                buf = response.read(4096)
241            lf.close()
242            ok = True
243        except EnvironmentError, e:
244            failed_exception = e
245            retries += 1
246        except httplib.HTTPException, e:
247            failed_exception = e
248            retries += 1
249        except sslerror, e:
250            failed_exception = e
251            retries += 1
252        except SSLError, e:
253            failed_exception = e
254            retries += 1
255
256    if retries > max_retries and failed_exception:
257        if log:
258            log.debug('Raising %s', failed_exception)
259        raise failed_excpetion
260
261# Functions to manipulate composite testbed names
262def testbed_base(tb):
263    """
264    Simple code to get the base testebd name.
265    """
266    i = tb.find('/')
267    if i == -1: return tb
268    else: return tb[0:i]
269
270def testbed_suffix(tb):
271    """
272    Simple code to get a testbed suffix, if nay.  No suffix returns None.
273    """
274    i = tb.find('/')
275    if i != -1: return tb[i+1:]
276    else: return None
277
278def split_testbed(tb):
279    """
280    Return a testbed and a suffix as a tuple.  No suffix returns None for that
281    field
282    """
283
284    i = tb.find('/')
285    if i != -1: return (tb[0:i], tb[i+1:])
286    else: return (tb, None)
287
288def join_testbed(base, suffix=None):
289    """
290    Build a testbed with suffix.  If base is itself a tuple, combine them,
291    otherwise combine the two.
292    """
293    if isinstance(base, tuple):
294        if len(base) == 2:
295            return '/'.join(base)
296        else:
297            raise RuntimeError("Too many tuple elements for join_testbed")
298    else:
299        if suffix:
300            return '/'.join((base, suffix))
301        else:
302            return base
303
304def abac_pem_type(cert):
305    key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$')
306    cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') 
307    type = None
308
309    f = open(cert, 'r')
310    for line in f:
311        if key_re.match(line):
312            if type is None: type = 'key'
313            elif type == 'cert': type = 'both'
314        elif cert_re.match(line):
315            if type is None: type = 'cert'
316            elif type == 'key': type = 'both'
317        if type == 'both': break
318    f.close()
319    return type
320
321def abac_split_cert(cert, keyfile=None, certfile=None):
322    """
323    Split the certificate file in cert into a certificate file and a key file
324    in cf and kf respectively.  The ABAC tools generally cannot handle combined
325    certificates/keys.  If kf anc cf are given, they are used, otherwise tmp
326    files are created.  Created tmp files must be deleted.  Problems opening or
327    writing files will cause exceptions.
328    """
329    class diversion:
330        '''
331        Wraps up the reqular expression to start and end a diversion, as well as
332        the open file that gets the lines.  If fd is passed in, use that system
333        file (probably from a mkstemp.  Otherwise open the given filename.
334        '''
335        def __init__(self, start, end, fn=None, fd=None):
336            self.start = re.compile(start)
337            self.end = re.compile(end)
338
339            if not fd:
340                # Open the file securely with minimal permissions. NB file
341                # cannot exist before this call.
342                fd = os.open(fn,
343                    (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600)
344
345            self.f = os.fdopen(fd, 'w')
346
347        def close(self):
348            self.f.close()
349
350    if not keyfile:
351        kf, rkeyfile = mkstemp(suffix=".pem")
352    else:
353        kf, rkeyfile = None, keyfile
354
355    if not certfile:
356        cf, rcertfile = mkstemp(suffix=".pem")
357    else:
358        cf, rcertfile = None, certfile
359
360    # Initialize the diversions
361    divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in (
362        ('\s*-----BEGIN RSA PRIVATE KEY-----$', 
363            '\s*-----END RSA PRIVATE KEY-----$',
364            keyfile, kf),
365        ('\s*-----BEGIN CERTIFICATE-----$', 
366            '\s*-----END CERTIFICATE-----$',
367            certfile, cf))]
368
369    # walk through the file, beginning a diversion when a start regexp
370    # matches until the end regexp matches.  While in the two regexps,
371    # print each line to the open diversion file (including the two
372    # matches).
373    active = None
374    f = open(cert, 'r')
375    for l in f:
376        if active:
377            if active.end.match(l):
378                print >>active.f, l,
379                active = None
380        else:
381            for d in divs:
382                if d.start.match(l):
383                    active = d
384                    break
385        if active: print >>active.f, l,
386
387    # This is probably unnecessary.  Close all the diversion files.
388    for d in divs: d.close()
389    return rkeyfile, rcertfile
390
391def abac_context_to_creds(context):
392    """
393    Pull all the credentials out of the context and return 2  lists of the
394    underlying credentials in an exportable format, IDs and attributes.
395    There are no duplicates in the lists.
396    """
397    ids, attrs = set(), set()
398    # This should be a one-iteration loop
399    for c in context.credentials():
400        ids.add(str(c.issuer_cert()))
401        attrs.add(str(c.attribute_cert()))
402
403    return list(ids), list(attrs)
404
405def find_pickle_problem(o, st=None):
406    """
407    Debugging routine to figure out what doesn't pickle from a dict full of
408    dicts and lists.  It tries to walk down the lists and dicts and pickle each
409    atom.  If something fails to pickle, it prints an approximation of a stack
410    trace through the data structure.
411    """
412    if st is None: st = [ ]
413
414    if isinstance(o, dict):
415        for k, i in o.items():
416            st.append(k)
417            find_pickle_problem(i, st)
418            st.pop()
419    elif isinstance(o, list):
420        st.append('list')
421        for i in o:
422            find_pickle_problem(i, st)
423        st.pop()
424    else:
425        try:
426            pickle.dumps(o)
427        except pickle.PicklingError, e:
428            print >>sys.stderr, "<PicklingError>"
429            print >>sys.stderr, st
430            print >>sys.stderr, o
431            print >>sys.stderr, e
432            print >>sys.stderr, "</PicklingError>"
433
Note: See TracBrowser for help on using the repository browser.