source: fedd/federation/util.py @ 19d2b72

Last change on this file since 19d2b72 was 19d2b72, checked in by Ted Faber <faber@…>, 11 years ago

Comment typo

  • Property mode set to 100644
File size: 12.2 KB
Line 
1#!/usr/local/bin/python
2
3import re
4import os
5import string
6import logging
7import pickle
8
9import httplib
10
11from optparse import OptionParser
12
13from socket import sslerror
14from tempfile import mkstemp
15
16from M2Crypto import SSL
17from M2Crypto.SSL import SSLError
18from deter import fedid
19from service_error import service_error
20from urlparse import urlparse
21from M2Crypto import m2
22
23
24# If this is an old enough version of M2Crypto.SSL that has an
25# ssl_verify_callback that doesn't allow 0-length signed certs, create a
26# version of that callback that does.  This is edited from the original in
27# M2Crypto.SSL.cb.  This version also elides the printing to stderr.
28if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None):
29    from M2Crypto.SSL.Context import map
30
31    def fedd_ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
32        unknown_issuer = [
33            m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY,
34            m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE,
35            m2.X509_V_ERR_CERT_UNTRUSTED,
36            m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
37            ]
38        # m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
39        if getattr(m2, 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None):
40            unknown_issuer.append(getattr(m2, 
41                'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None))
42        ssl_ctx = map()[ssl_ctx_ptr]
43
44        if errnum in unknown_issuer: 
45            if ssl_ctx.get_allow_unknown_ca():
46                ok = 1
47        # CRL checking goes here...
48        if ok:
49            if ssl_ctx.get_verify_depth() >= errdepth:
50                ok = 1
51            else:
52                ok = 0
53        return ok
54else:
55    def fedd_ssl_verify_callback(ok, store):
56        '''
57        m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
58        '''
59        errnum = store.get_error()
60        if errnum == m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
61            ok = 1
62            return ok
63        else:
64            return SSL.cb.ssl_verify_callback_allow_unknown_ca(ok, store)
65
66class fedd_ssl_context(SSL.Context):
67    """
68    Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd.
69    """
70    def __init__(self, my_cert, trusted_certs=None, password=None, protocol=None):
71        """
72        construct a fedd_ssl_context
73
74        @param my_cert: PEM file with my certificate in it
75        @param trusted_certs: PEM file with trusted certs in it (optional)
76        """
77        if protocol: SSL.Context.__init__(self, protocol=protocol)
78        else: SSL.Context.__init__(self)
79
80        # load_cert takes a callback to get a password, not a password, so if
81        # the caller provided a password, this creates a nonce callback using a
82        # lambda form.
83        if password != None and not callable(password):
84            # This is cute.  password = lambda *args: password produces a
85            # function object that returns itself rather than one that returns
86            # the object itself.  This is because password is an object
87            # reference and after the assignment it's a lambda.  So we assign
88            # to a temp.
89            pwd = str(password)
90            password =lambda *args: pwd
91
92        # The calls to str below (and above) are because the underlying SSL
93        # stuff is intolerant of unicode.
94        if password != None:
95            self.load_cert(str(my_cert), callback=password)
96        else:
97            self.load_cert(str(my_cert))
98
99        # If no trusted certificates are specified, allow unknown CAs.
100        if trusted_certs: 
101            self.load_verify_locations(trusted_certs)
102            self.set_verify(SSL.verify_peer, 10)
103        else:
104            # Install the proper callback to allow self-signed certs
105            self.set_allow_unknown_ca(True)
106            self.set_verify(SSL.verify_peer, 10, 
107                    callback=fedd_ssl_verify_callback)
108
109        # no session caching
110        self.set_session_cache_mode(0)
111
112class file_expanding_opts(OptionParser):
113    def expand_file(self, option, opt_str, v, p):
114        """
115        Store the given value to the given destination after expanding home
116        directories.
117        """
118        setattr(p.values, option.dest, os.path.expanduser(v))
119
120    def __init__(self, usage=None, version=None):
121        OptionParser.__init__(self)
122
123
124def read_simple_accessdb(fn, auth, mask=[]):
125    """
126    Read a simple access database.  Each line is a fedid (of the form
127    fedid:hexstring) and a comma separated list of atributes to be assigned to
128    it.  This parses out the fedids and adds the attributes to the authorizer.
129    comments (preceded with a #) and blank lines are ignored.  Exceptions (e.g.
130    file exceptions and ValueErrors from badly parsed lines) are propagated.
131    """
132
133    rv = [ ]
134    lineno = 0
135    fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\
136            "(\w+\s*(,\s*\w+)*)")
137
138    # If a single string came in, make it a list
139    if isinstance(mask, basestring): mask = [ mask ]
140
141    f = open(fn, 'r')
142    for line in f:
143        lineno += 1
144        line = line.strip()
145        if line.startswith('#') or len(line) == 0: 
146            continue
147        m = fedid_line.match(line)
148        if m :
149            fid = fedid(hexstr=m.group(1))
150            for a in [ a.strip() for a in m.group(2).split(",") \
151                    if not mask or a.strip() in mask ]:
152                auth.set_attribute(fid, a.strip())
153        else:
154            raise ValueError("Badly formatted line in accessdb: %s line %d" %\
155                    (fn, lineno))
156    f.close()
157    return rv
158       
159
160def pack_id(id):
161    """
162    Return a dictionary with the field name set by the id type.  Handy for
163    creating dictionaries to be converted to messages.
164    """
165    if isinstance(id, fedid): return { 'fedid': id }
166    elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id }
167    else: return { 'localname': id}
168
169def unpack_id(id):
170    """return id as a type determined by the key"""
171    for k in ("localname", "fedid", "uri", "kerberosUsername"):
172        if id.has_key(k): return id[k]
173    return None
174
175def set_log_level(config, sect, log):
176    """ Set the logging level to the value passed in sect of config."""
177    # The getattr sleight of hand finds the logging level constant
178    # corrersponding to the string.  We're a little paranoid to avoid user
179    # mayhem.
180    if config.has_option(sect, "log_level"):
181        level_str = config.get(sect, "log_level")
182        try:
183            level = int(getattr(logging, level_str.upper(), -1))
184
185            if  logging.DEBUG <= level <= logging.CRITICAL:
186                log.setLevel(level)
187            else:
188                log.error("Bad experiment_log value: %s" % level_str)
189
190        except ValueError:
191            log.error("Bad experiment_log value: %s" % level_str)
192
193def copy_file(src, dest, size=1024):
194    """
195    Exceedingly simple file copy.  Throws an EnvironmentError if there's a problem.
196    """
197    s = open(src,'r')
198    d = open(dest, 'w')
199
200    buf = s.read(size)
201    while buf != "":
202        d.write(buf)
203        buf = s.read(size)
204    s.close()
205    d.close()
206
207def get_url(url, cf, destdir, fn=None, max_retries=5, log=None):
208    """
209    Get data from a federated data store.  This presents the client cert/fedid
210    to the http server.  We retry up to max_retries times.
211    """
212    po = urlparse(url)
213    if not fn:
214        fn = po.path.rpartition('/')[2]
215    retries = 0
216    ok = False
217    failed_exception = None
218    while not ok and retries < max_retries:
219        try:
220            conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
221                    cert_file=cf, key_file=cf, timeout=30)
222            conn.putrequest('GET', po.path)
223            conn.endheaders()
224            response = conn.getresponse()
225
226            lf = open("%s/%s" % (destdir, fn), "w")
227            buf = response.read(4096)
228            while buf:
229                lf.write(buf)
230                buf = response.read(4096)
231            lf.close()
232            ok = True
233        except EnvironmentError, e:
234            failed_exception = e
235            retries += 1
236        except httplib.HTTPException, e:
237            failed_exception = e
238            retries += 1
239        except sslerror, e:
240            failed_exception = e
241            retries += 1
242        except SSLError, e:
243            failed_exception = e
244            retries += 1
245
246    if retries > max_retries and failed_exception:
247        if log:
248            log.debug('Raising %s', failed_exception)
249        raise failed_excpetion
250
251# Functions to manipulate composite testbed names
252def testbed_base(tb):
253    """
254    Simple code to get the base testebd name.
255    """
256    i = tb.find('/')
257    if i == -1: return tb
258    else: return tb[0:i]
259
260def testbed_suffix(tb):
261    """
262    Simple code to get a testbed suffix, if any.  No suffix returns None.
263    """
264    i = tb.find('/')
265    if i != -1: return tb[i+1:]
266    else: return None
267
268def split_testbed(tb):
269    """
270    Return a testbed and a suffix as a tuple.  No suffix returns None for that
271    field
272    """
273
274    i = tb.find('/')
275    if i != -1: return (tb[0:i], tb[i+1:])
276    else: return (tb, None)
277
278def join_testbed(base, suffix=None):
279    """
280    Build a testbed with suffix.  If base is itself a tuple, combine them,
281    otherwise combine the two.
282    """
283    if isinstance(base, tuple):
284        if len(base) == 2:
285            return '/'.join(base)
286        else:
287            raise RuntimeError("Too many tuple elements for join_testbed")
288    else:
289        if suffix:
290            return '/'.join((base, suffix))
291        else:
292            return base
293
294def abac_pem_type(cert):
295    key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$')
296    cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') 
297    type = None
298
299    f = open(cert, 'r')
300    for line in f:
301        if key_re.match(line):
302            if type is None: type = 'key'
303            elif type == 'cert': type = 'both'
304        elif cert_re.match(line):
305            if type is None: type = 'cert'
306            elif type == 'key': type = 'both'
307        if type == 'both': break
308    f.close()
309    return type
310
311def abac_split_cert(cert, keyfile=None, certfile=None):
312    """
313    Split the certificate file in cert into a certificate file and a key file
314    in cf and kf respectively.  The ABAC tools generally cannot handle combined
315    certificates/keys.  If kf anc cf are given, they are used, otherwise tmp
316    files are created.  Created tmp files must be deleted.  Problems opening or
317    writing files will cause exceptions.
318    """
319    class diversion:
320        '''
321        Wraps up the reqular expression to start and end a diversion, as well as
322        the open file that gets the lines.  If fd is passed in, use that system
323        file (probably from a mkstemp.  Otherwise open the given filename.
324        '''
325        def __init__(self, start, end, fn=None, fd=None):
326            self.start = re.compile(start)
327            self.end = re.compile(end)
328
329            if not fd:
330                # Open the file securely with minimal permissions. NB file
331                # cannot exist before this call.
332                fd = os.open(fn,
333                    (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600)
334
335            self.f = os.fdopen(fd, 'w')
336
337        def close(self):
338            self.f.close()
339
340    if not keyfile:
341        kf, rkeyfile = mkstemp(suffix=".pem")
342    else:
343        kf, rkeyfile = None, keyfile
344
345    if not certfile:
346        cf, rcertfile = mkstemp(suffix=".pem")
347    else:
348        cf, rcertfile = None, certfile
349
350    # Initialize the diversions
351    divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in (
352        ('\s*-----BEGIN RSA PRIVATE KEY-----$', 
353            '\s*-----END RSA PRIVATE KEY-----$',
354            keyfile, kf),
355        ('\s*-----BEGIN CERTIFICATE-----$', 
356            '\s*-----END CERTIFICATE-----$',
357            certfile, cf))]
358
359    # walk through the file, beginning a diversion when a start regexp
360    # matches until the end regexp matches.  While in the two regexps,
361    # print each line to the open diversion file (including the two
362    # matches).
363    active = None
364    f = open(cert, 'r')
365    for l in f:
366        if active:
367            if active.end.match(l):
368                print >>active.f, l,
369                active = None
370        else:
371            for d in divs:
372                if d.start.match(l):
373                    active = d
374                    break
375        if active: print >>active.f, l,
376
377    # This is probably unnecessary.  Close all the diversion files.
378    for d in divs: d.close()
379    return rkeyfile, rcertfile
380
381def abac_context_to_creds(context):
382    """
383    Pull all the credentials out of the context and return 2  lists of the
384    underlying credentials in an exportable format, IDs and attributes.
385    There are no duplicates in the lists.
386    """
387    ids, attrs = set(), set()
388    # This should be a one-iteration loop
389    for c in context.credentials():
390        ids.add(str(c.issuer_cert()))
391        attrs.add(str(c.attribute_cert()))
392
393    return list(ids), list(attrs)
394
395def find_pickle_problem(o, st=None):
396    """
397    Debugging routine to figure out what doesn't pickle from a dict full of
398    dicts and lists.  It tries to walk down the lists and dicts and pickle each
399    atom.  If something fails to pickle, it prints an approximation of a stack
400    trace through the data structure.
401    """
402    if st is None: st = [ ]
403
404    if isinstance(o, dict):
405        for k, i in o.items():
406            st.append(k)
407            find_pickle_problem(i, st)
408            st.pop()
409    elif isinstance(o, list):
410        st.append('list')
411        for i in o:
412            find_pickle_problem(i, st)
413        st.pop()
414    else:
415        try:
416            pickle.dumps(o)
417        except pickle.PicklingError, e:
418            print >>sys.stderr, "<PicklingError>"
419            print >>sys.stderr, st
420            print >>sys.stderr, o
421            print >>sys.stderr, e
422            print >>sys.stderr, "</PicklingError>"
423
Note: See TracBrowser for help on using the repository browser.