source: fedd/federation/util.py @ 661e857

Last change on this file since 661e857 was 661e857, checked in by Ted Faber <faber@…>, 12 years ago

Remove excessive debug logging (yep it exists)

  • Property mode set to 100644
File size: 12.3 KB
Line 
1#!/usr/local/bin/python
2
3import re
4import os
5import string
6import logging
7import pickle
8
9import httplib
10
11from optparse import OptionParser
12
13from socket import sslerror
14from tempfile import mkstemp
15
16from M2Crypto import SSL
17from M2Crypto.SSL import SSLError
18from deter import fedid
19from service_error import service_error
20from urlparse import urlparse
21from M2Crypto import m2
22
23
24# If this is an old enough version of M2Crypto.SSL that has an
25# ssl_verify_callback that doesn't allow 0-length signed certs, create a
26# version of that callback that does.  This is edited from the original in
27# M2Crypto.SSL.cb.  This version also elides the printing to stderr.
28if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None):
29    from M2Crypto.SSL.Context import map
30
31    def fedd_ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
32        unknown_issuer = [
33            m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY,
34            m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE,
35            m2.X509_V_ERR_CERT_UNTRUSTED,
36            m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
37            ]
38        # m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
39        if getattr(m2, 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None):
40            unknown_issuer.append(getattr(m2, 
41                'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None))
42        ssl_ctx = map()[ssl_ctx_ptr]
43
44        if errnum in unknown_issuer: 
45            if ssl_ctx.get_allow_unknown_ca():
46                ok = 1
47        # CRL checking goes here...
48        if ok:
49            if ssl_ctx.get_verify_depth() >= errdepth:
50                ok = 1
51            else:
52                ok = 0
53        return ok
54else:
55    def fedd_ssl_verify_callback(ok, store):
56        '''
57        m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
58        '''
59        errnum = store.get_error()
60        if errnum == m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
61            ok = 1
62            return ok
63        else:
64            return SSL.cb.ssl_verify_callback_allow_unknown_ca(ok, store)
65
66class fedd_ssl_context(SSL.Context):
67    """
68    Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd.
69    """
70    def __init__(self, my_cert, trusted_certs=None, password=None, protocol=None):
71        """
72        construct a fedd_ssl_context
73
74        @param my_cert: PEM file with my certificate in it
75        @param trusted_certs: PEM file with trusted certs in it (optional)
76        """
77        if protocol: SSL.Context.__init__(self, protocol=protocol)
78        else: SSL.Context.__init__(self)
79
80        # load_cert takes a callback to get a password, not a password, so if
81        # the caller provided a password, this creates a nonce callback using a
82        # lambda form.
83        if password != None and not callable(password):
84            # This is cute.  password = lambda *args: password produces a
85            # function object that returns itself rather than one that returns
86            # the object itself.  This is because password is an object
87            # reference and after the assignment it's a lambda.  So we assign
88            # to a temp.
89            pwd = str(password)
90            password =lambda *args: pwd
91
92        # The calls to str below (and above) are because the underlying SSL
93        # stuff is intolerant of unicode.
94        if password != None:
95            self.load_cert(str(my_cert), callback=password)
96        else:
97            self.load_cert(str(my_cert))
98
99        # If no trusted certificates are specified, allow unknown CAs.
100        if trusted_certs: 
101            self.load_verify_locations(trusted_certs)
102            self.set_verify(SSL.verify_peer, 10)
103        else:
104            # Install the proper callback to allow self-signed certs
105            self.set_allow_unknown_ca(True)
106            self.set_verify(SSL.verify_peer, 10, 
107                    callback=fedd_ssl_verify_callback)
108
109        # no session caching
110        self.set_session_cache_mode(0)
111
112class file_expanding_opts(OptionParser):
113    def expand_file(self, option, opt_str, v, p):
114        """
115        Store the given value to the given destination after expanding home
116        directories.
117        """
118        setattr(p.values, option.dest, os.path.expanduser(v))
119
120    def __init__(self, usage=None, version=None):
121        OptionParser.__init__(self)
122
123
124def read_simple_accessdb(fn, auth, mask=[]):
125    """
126    Read a simple access database.  Each line is a fedid (of the form
127    fedid:hexstring) and a comma separated list of atributes to be assigned to
128    it.  This parses out the fedids and adds the attributes to the authorizer.
129    comments (preceded with a #) and blank lines are ignored.  Exceptions (e.g.
130    file exceptions and ValueErrors from badly parsed lines) are propagated.
131    """
132
133    rv = [ ]
134    lineno = 0
135    fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\
136            "(\w+\s*(,\s*\w+)*)")
137
138    # If a single string came in, make it a list
139    if isinstance(mask, basestring): mask = [ mask ]
140
141    f = open(fn, 'r')
142    for line in f:
143        lineno += 1
144        line = line.strip()
145        if line.startswith('#') or len(line) == 0: 
146            continue
147        m = fedid_line.match(line)
148        if m :
149            fid = fedid(hexstr=m.group(1))
150            for a in [ a.strip() for a in m.group(2).split(",") \
151                    if not mask or a.strip() in mask ]:
152                auth.set_attribute(fid, a.strip())
153        else:
154            raise ValueError("Badly formatted line in accessdb: %s line %d" %\
155                    (fn, lineno))
156    f.close()
157    return rv
158       
159
160def pack_id(id):
161    """
162    Return a dictionary with the field name set by the id type.  Handy for
163    creating dictionaries to be converted to messages.
164    """
165    if isinstance(id, fedid): return { 'fedid': id }
166    elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id }
167    else: return { 'localname': id}
168
169def unpack_id(id):
170    """return id as a type determined by the key"""
171    for k in ("localname", "fedid", "uri", "kerberosUsername"):
172        if id.has_key(k): return id[k]
173    return None
174
175def set_log_level(config, sect, log):
176    """ Set the logging level to the value passed in sect of config."""
177    # The getattr sleight of hand finds the logging level constant
178    # corrersponding to the string.  We're a little paranoid to avoid user
179    # mayhem.
180    if config.has_option(sect, "log_level"):
181        level_str = config.get(sect, "log_level")
182        try:
183            level = int(getattr(logging, level_str.upper(), -1))
184
185            if  logging.DEBUG <= level <= logging.CRITICAL:
186                log.setLevel(level)
187            else:
188                log.error("Bad experiment_log value: %s" % level_str)
189
190        except ValueError:
191            log.error("Bad experiment_log value: %s" % level_str)
192
193def copy_file(src, dest, size=1024):
194    """
195    Exceedingly simple file copy.  Throws an EnvironmentError if there's a problem.
196    """
197    s = open(src,'r')
198    d = open(dest, 'w')
199
200    buf = s.read(size)
201    while buf != "":
202        d.write(buf)
203        buf = s.read(size)
204    s.close()
205    d.close()
206
207def get_url(url, cf, destdir, fn=None, max_retries=5, log=None):
208    """
209    Get data from a federated data store.  This presents the client cert/fedid
210    to the http server.  We retry up to max_retries times.
211    """
212    po = urlparse(url)
213    if not fn:
214        fn = po.path.rpartition('/')[2]
215    retries = 0
216    ok = False
217    failed_exception = None
218    while not ok and retries < max_retries:
219        try:
220            if log:
221                log.debug('Creating HTTPSConnection: %s' % cf)
222            conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
223                    cert_file=cf, key_file=cf, timeout=30)
224            conn.putrequest('GET', po.path)
225            conn.endheaders()
226            response = conn.getresponse()
227
228            lf = open("%s/%s" % (destdir, fn), "w")
229            buf = response.read(4096)
230            while buf:
231                lf.write(buf)
232                buf = response.read(4096)
233            lf.close()
234            ok = True
235        except EnvironmentError, e:
236            failed_exception = e
237            retries += 1
238        except httplib.HTTPException, e:
239            failed_exception = e
240            retries += 1
241        except sslerror, e:
242            failed_exception = e
243            retries += 1
244        except SSLError, e:
245            failed_exception = e
246            retries += 1
247
248    if retries > max_retries and failed_exception:
249        if log:
250            log.debug('Raising %s', failed_exception)
251        raise failed_excpetion
252
253# Functions to manipulate composite testbed names
254def testbed_base(tb):
255    """
256    Simple code to get the base testebd name.
257    """
258    i = tb.find('/')
259    if i == -1: return tb
260    else: return tb[0:i]
261
262def testbed_suffix(tb):
263    """
264    Simple code to get a testbed suffix, if nay.  No suffix returns None.
265    """
266    i = tb.find('/')
267    if i != -1: return tb[i+1:]
268    else: return None
269
270def split_testbed(tb):
271    """
272    Return a testbed and a suffix as a tuple.  No suffix returns None for that
273    field
274    """
275
276    i = tb.find('/')
277    if i != -1: return (tb[0:i], tb[i+1:])
278    else: return (tb, None)
279
280def join_testbed(base, suffix=None):
281    """
282    Build a testbed with suffix.  If base is itself a tuple, combine them,
283    otherwise combine the two.
284    """
285    if isinstance(base, tuple):
286        if len(base) == 2:
287            return '/'.join(base)
288        else:
289            raise RuntimeError("Too many tuple elements for join_testbed")
290    else:
291        if suffix:
292            return '/'.join((base, suffix))
293        else:
294            return base
295
296def abac_pem_type(cert):
297    key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$')
298    cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') 
299    type = None
300
301    f = open(cert, 'r')
302    for line in f:
303        if key_re.match(line):
304            if type is None: type = 'key'
305            elif type == 'cert': type = 'both'
306        elif cert_re.match(line):
307            if type is None: type = 'cert'
308            elif type == 'key': type = 'both'
309        if type == 'both': break
310    f.close()
311    return type
312
313def abac_split_cert(cert, keyfile=None, certfile=None):
314    """
315    Split the certificate file in cert into a certificate file and a key file
316    in cf and kf respectively.  The ABAC tools generally cannot handle combined
317    certificates/keys.  If kf anc cf are given, they are used, otherwise tmp
318    files are created.  Created tmp files must be deleted.  Problems opening or
319    writing files will cause exceptions.
320    """
321    class diversion:
322        '''
323        Wraps up the reqular expression to start and end a diversion, as well as
324        the open file that gets the lines.  If fd is passed in, use that system
325        file (probably from a mkstemp.  Otherwise open the given filename.
326        '''
327        def __init__(self, start, end, fn=None, fd=None):
328            self.start = re.compile(start)
329            self.end = re.compile(end)
330
331            if not fd:
332                # Open the file securely with minimal permissions. NB file
333                # cannot exist before this call.
334                fd = os.open(fn,
335                    (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600)
336
337            self.f = os.fdopen(fd, 'w')
338
339        def close(self):
340            self.f.close()
341
342    if not keyfile:
343        kf, rkeyfile = mkstemp(suffix=".pem")
344    else:
345        kf, rkeyfile = None, keyfile
346
347    if not certfile:
348        cf, rcertfile = mkstemp(suffix=".pem")
349    else:
350        cf, rcertfile = None, certfile
351
352    # Initialize the diversions
353    divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in (
354        ('\s*-----BEGIN RSA PRIVATE KEY-----$', 
355            '\s*-----END RSA PRIVATE KEY-----$',
356            keyfile, kf),
357        ('\s*-----BEGIN CERTIFICATE-----$', 
358            '\s*-----END CERTIFICATE-----$',
359            certfile, cf))]
360
361    # walk through the file, beginning a diversion when a start regexp
362    # matches until the end regexp matches.  While in the two regexps,
363    # print each line to the open diversion file (including the two
364    # matches).
365    active = None
366    f = open(cert, 'r')
367    for l in f:
368        if active:
369            if active.end.match(l):
370                print >>active.f, l,
371                active = None
372        else:
373            for d in divs:
374                if d.start.match(l):
375                    active = d
376                    break
377        if active: print >>active.f, l,
378
379    # This is probably unnecessary.  Close all the diversion files.
380    for d in divs: d.close()
381    return rkeyfile, rcertfile
382
383def abac_context_to_creds(context):
384    """
385    Pull all the credentials out of the context and return 2  lists of the
386    underlying credentials in an exportable format, IDs and attributes.
387    There are no duplicates in the lists.
388    """
389    ids, attrs = set(), set()
390    # This should be a one-iteration loop
391    for c in context.credentials():
392        ids.add(str(c.issuer_cert()))
393        attrs.add(str(c.attribute_cert()))
394
395    return list(ids), list(attrs)
396
397def find_pickle_problem(o, st=None):
398    """
399    Debugging routine to figure out what doesn't pickle from a dict full of
400    dicts and lists.  It tries to walk down the lists and dicts and pickle each
401    atom.  If something fails to pickle, it prints an approximation of a stack
402    trace through the data structure.
403    """
404    if st is None: st = [ ]
405
406    if isinstance(o, dict):
407        for k, i in o.items():
408            st.append(k)
409            find_pickle_problem(i, st)
410            st.pop()
411    elif isinstance(o, list):
412        st.append('list')
413        for i in o:
414            find_pickle_problem(i, st)
415        st.pop()
416    else:
417        try:
418            pickle.dumps(o)
419        except pickle.PicklingError, e:
420            print >>sys.stderr, "<PicklingError>"
421            print >>sys.stderr, st
422            print >>sys.stderr, o
423            print >>sys.stderr, e
424            print >>sys.stderr, "</PicklingError>"
425
Note: See TracBrowser for help on using the repository browser.