source: fedd/federation/util.py @ c3a573c

Last change on this file since c3a573c was c3a573c, checked in by Ted Faber <faber@…>, 10 years ago

Defensive driving around new ssl libraries. Fix error handling in get_url

  • Property mode set to 100644
File size: 12.7 KB
Line 
1#!/usr/local/bin/python
2
3import re
4import os
5import string
6import logging
7import pickle
8
9import httplib
10import ssl
11
12from optparse import OptionParser
13
14from socket import sslerror
15from tempfile import mkstemp
16
17from M2Crypto import SSL
18from M2Crypto.SSL import SSLError
19from deter import fedid
20from service_error import service_error
21from urlparse import urlparse
22from M2Crypto import m2
23
24
25# If this is an old enough version of M2Crypto.SSL that has an
26# ssl_verify_callback that doesn't allow 0-length signed certs, create a
27# version of that callback that does.  This is edited from the original in
28# M2Crypto.SSL.cb.  This version also elides the printing to stderr.
29if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None):
30    from M2Crypto.SSL.Context import map
31
32    def fedd_ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
33        unknown_issuer = [
34            m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY,
35            m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE,
36            m2.X509_V_ERR_CERT_UNTRUSTED,
37            m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
38            ]
39        # m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
40        if getattr(m2, 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None):
41            unknown_issuer.append(getattr(m2, 
42                'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None))
43        ssl_ctx = map()[ssl_ctx_ptr]
44
45        if errnum in unknown_issuer: 
46            if ssl_ctx.get_allow_unknown_ca():
47                ok = 1
48        # CRL checking goes here...
49        if ok:
50            if ssl_ctx.get_verify_depth() >= errdepth:
51                ok = 1
52            else:
53                ok = 0
54        return ok
55else:
56    def fedd_ssl_verify_callback(ok, store):
57        '''
58        m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
59        '''
60        errnum = store.get_error()
61        if errnum == m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
62            ok = 1
63            return ok
64        else:
65            return SSL.cb.ssl_verify_callback_allow_unknown_ca(ok, store)
66
67class fedd_ssl_context(SSL.Context):
68    """
69    Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd.
70    """
71    def __init__(self, my_cert, trusted_certs=None, password=None, protocol=None):
72        """
73        construct a fedd_ssl_context
74
75        @param my_cert: PEM file with my certificate in it
76        @param trusted_certs: PEM file with trusted certs in it (optional)
77        """
78        if protocol: SSL.Context.__init__(self, protocol=protocol)
79        else: SSL.Context.__init__(self)
80
81        # load_cert takes a callback to get a password, not a password, so if
82        # the caller provided a password, this creates a nonce callback using a
83        # lambda form.
84        if password != None and not callable(password):
85            # This is cute.  password = lambda *args: password produces a
86            # function object that returns itself rather than one that returns
87            # the object itself.  This is because password is an object
88            # reference and after the assignment it's a lambda.  So we assign
89            # to a temp.
90            pwd = str(password)
91            password =lambda *args: pwd
92
93        # The calls to str below (and above) are because the underlying SSL
94        # stuff is intolerant of unicode.
95        if password != None:
96            self.load_cert(str(my_cert), callback=password)
97        else:
98            self.load_cert(str(my_cert))
99
100        # If no trusted certificates are specified, allow unknown CAs.
101        if trusted_certs: 
102            self.load_verify_locations(trusted_certs)
103            self.set_verify(SSL.verify_peer, 10)
104        else:
105            # Install the proper callback to allow self-signed certs
106            self.set_allow_unknown_ca(True)
107            self.set_verify(SSL.verify_peer, 10, 
108                    callback=fedd_ssl_verify_callback)
109
110        # no session caching
111        self.set_session_cache_mode(0)
112
113class file_expanding_opts(OptionParser):
114    def expand_file(self, option, opt_str, v, p):
115        """
116        Store the given value to the given destination after expanding home
117        directories.
118        """
119        setattr(p.values, option.dest, os.path.expanduser(v))
120
121    def __init__(self, usage=None, version=None):
122        OptionParser.__init__(self)
123
124
125def read_simple_accessdb(fn, auth, mask=[]):
126    """
127    Read a simple access database.  Each line is a fedid (of the form
128    fedid:hexstring) and a comma separated list of atributes to be assigned to
129    it.  This parses out the fedids and adds the attributes to the authorizer.
130    comments (preceded with a #) and blank lines are ignored.  Exceptions (e.g.
131    file exceptions and ValueErrors from badly parsed lines) are propagated.
132    """
133
134    rv = [ ]
135    lineno = 0
136    fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\
137            "(\w+\s*(,\s*\w+)*)")
138
139    # If a single string came in, make it a list
140    if isinstance(mask, basestring): mask = [ mask ]
141
142    f = open(fn, 'r')
143    for line in f:
144        lineno += 1
145        line = line.strip()
146        if line.startswith('#') or len(line) == 0: 
147            continue
148        m = fedid_line.match(line)
149        if m :
150            fid = fedid(hexstr=m.group(1))
151            for a in [ a.strip() for a in m.group(2).split(",") \
152                    if not mask or a.strip() in mask ]:
153                auth.set_attribute(fid, a.strip())
154        else:
155            raise ValueError("Badly formatted line in accessdb: %s line %d" %\
156                    (fn, lineno))
157    f.close()
158    return rv
159       
160
161def pack_id(id):
162    """
163    Return a dictionary with the field name set by the id type.  Handy for
164    creating dictionaries to be converted to messages.
165    """
166    if isinstance(id, fedid): return { 'fedid': id }
167    elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id }
168    else: return { 'localname': id}
169
170def unpack_id(id):
171    """return id as a type determined by the key"""
172    for k in ("localname", "fedid", "uri", "kerberosUsername"):
173        if id.has_key(k): return id[k]
174    return None
175
176def set_log_level(config, sect, log):
177    """ Set the logging level to the value passed in sect of config."""
178    # The getattr sleight of hand finds the logging level constant
179    # corrersponding to the string.  We're a little paranoid to avoid user
180    # mayhem.
181    if config.has_option(sect, "log_level"):
182        level_str = config.get(sect, "log_level")
183        try:
184            level = int(getattr(logging, level_str.upper(), -1))
185
186            if  logging.DEBUG <= level <= logging.CRITICAL:
187                log.setLevel(level)
188            else:
189                log.error("Bad experiment_log value: %s" % level_str)
190
191        except ValueError:
192            log.error("Bad experiment_log value: %s" % level_str)
193
194def copy_file(src, dest, size=1024):
195    """
196    Exceedingly simple file copy.  Throws an EnvironmentError if there's a problem.
197    """
198    s = open(src,'r')
199    d = open(dest, 'w')
200
201    buf = s.read(size)
202    while buf != "":
203        d.write(buf)
204        buf = s.read(size)
205    s.close()
206    d.close()
207
208def get_url(url, cf, destdir, fn=None, max_retries=5, log=None):
209    """
210    Get data from a federated data store.  This presents the client cert/fedid
211    to the http server.  We retry up to max_retries times.
212    """
213    po = urlparse(url)
214    if not fn:
215        fn = po.path.rpartition('/')[2]
216    retries = 0
217    ok = False
218    failed_exception = None
219    # Build a context on modern systems, on old ones fall back to the old
220    # HTTPSConnection definition.
221    context = None
222    try:
223        # This context does not check the self-signed server cert
224        context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
225        context.load_cert_chain(cf)
226    except AttributeError:
227        pass
228    while not ok and retries < max_retries:
229        try:
230            if context is not None:
231                conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
232                        timeout=30, context=context)
233            else:
234                conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
235                        cert_file=cf, key_file=cf, timeout=30)
236            conn.putrequest('GET', po.path)
237            conn.endheaders()
238            response = conn.getresponse()
239
240            lf = open("%s/%s" % (destdir, fn), "w")
241            buf = response.read(4096)
242            while buf:
243                lf.write(buf)
244                buf = response.read(4096)
245            lf.close()
246            ok = True
247        except EnvironmentError, e:
248            failed_exception = e
249            retries += 1
250        except httplib.HTTPException, e:
251            failed_exception = e
252            retries += 1
253        except sslerror, e:
254            failed_exception = e
255            retries += 1
256        except SSLError, e:
257            failed_exception = e
258            retries += 1
259
260    if retries >= max_retries and failed_exception:
261        if log:
262            log.debug('Raising %s', failed_exception)
263        raise failed_exception
264
265# Functions to manipulate composite testbed names
266def testbed_base(tb):
267    """
268    Simple code to get the base testebd name.
269    """
270    i = tb.find('/')
271    if i == -1: return tb
272    else: return tb[0:i]
273
274def testbed_suffix(tb):
275    """
276    Simple code to get a testbed suffix, if any.  No suffix returns None.
277    """
278    i = tb.find('/')
279    if i != -1: return tb[i+1:]
280    else: return None
281
282def split_testbed(tb):
283    """
284    Return a testbed and a suffix as a tuple.  No suffix returns None for that
285    field
286    """
287
288    i = tb.find('/')
289    if i != -1: return (tb[0:i], tb[i+1:])
290    else: return (tb, None)
291
292def join_testbed(base, suffix=None):
293    """
294    Build a testbed with suffix.  If base is itself a tuple, combine them,
295    otherwise combine the two.
296    """
297    if isinstance(base, tuple):
298        if len(base) == 2:
299            return '/'.join(base)
300        else:
301            raise RuntimeError("Too many tuple elements for join_testbed")
302    else:
303        if suffix:
304            return '/'.join((base, suffix))
305        else:
306            return base
307
308def abac_pem_type(cert):
309    key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$')
310    cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') 
311    type = None
312
313    f = open(cert, 'r')
314    for line in f:
315        if key_re.match(line):
316            if type is None: type = 'key'
317            elif type == 'cert': type = 'both'
318        elif cert_re.match(line):
319            if type is None: type = 'cert'
320            elif type == 'key': type = 'both'
321        if type == 'both': break
322    f.close()
323    return type
324
325def abac_split_cert(cert, keyfile=None, certfile=None):
326    """
327    Split the certificate file in cert into a certificate file and a key file
328    in cf and kf respectively.  The ABAC tools generally cannot handle combined
329    certificates/keys.  If kf anc cf are given, they are used, otherwise tmp
330    files are created.  Created tmp files must be deleted.  Problems opening or
331    writing files will cause exceptions.
332    """
333    class diversion:
334        '''
335        Wraps up the reqular expression to start and end a diversion, as well as
336        the open file that gets the lines.  If fd is passed in, use that system
337        file (probably from a mkstemp.  Otherwise open the given filename.
338        '''
339        def __init__(self, start, end, fn=None, fd=None):
340            self.start = re.compile(start)
341            self.end = re.compile(end)
342
343            if not fd:
344                # Open the file securely with minimal permissions. NB file
345                # cannot exist before this call.
346                fd = os.open(fn,
347                    (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600)
348
349            self.f = os.fdopen(fd, 'w')
350
351        def close(self):
352            self.f.close()
353
354    if not keyfile:
355        kf, rkeyfile = mkstemp(suffix=".pem")
356    else:
357        kf, rkeyfile = None, keyfile
358
359    if not certfile:
360        cf, rcertfile = mkstemp(suffix=".pem")
361    else:
362        cf, rcertfile = None, certfile
363
364    # Initialize the diversions
365    divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in (
366        ('\s*-----BEGIN RSA PRIVATE KEY-----$', 
367            '\s*-----END RSA PRIVATE KEY-----$',
368            keyfile, kf),
369        ('\s*-----BEGIN CERTIFICATE-----$', 
370            '\s*-----END CERTIFICATE-----$',
371            certfile, cf))]
372
373    # walk through the file, beginning a diversion when a start regexp
374    # matches until the end regexp matches.  While in the two regexps,
375    # print each line to the open diversion file (including the two
376    # matches).
377    active = None
378    f = open(cert, 'r')
379    for l in f:
380        if active:
381            if active.end.match(l):
382                print >>active.f, l,
383                active = None
384        else:
385            for d in divs:
386                if d.start.match(l):
387                    active = d
388                    break
389        if active: print >>active.f, l,
390
391    # This is probably unnecessary.  Close all the diversion files.
392    for d in divs: d.close()
393    return rkeyfile, rcertfile
394
395def abac_context_to_creds(context):
396    """
397    Pull all the credentials out of the context and return 2  lists of the
398    underlying credentials in an exportable format, IDs and attributes.
399    There are no duplicates in the lists.
400    """
401    ids, attrs = set(), set()
402    # This should be a one-iteration loop
403    for c in context.credentials():
404        ids.add(str(c.issuer_cert()))
405        attrs.add(str(c.attribute_cert()))
406
407    return list(ids), list(attrs)
408
409def find_pickle_problem(o, st=None):
410    """
411    Debugging routine to figure out what doesn't pickle from a dict full of
412    dicts and lists.  It tries to walk down the lists and dicts and pickle each
413    atom.  If something fails to pickle, it prints an approximation of a stack
414    trace through the data structure.
415    """
416    if st is None: st = [ ]
417
418    if isinstance(o, dict):
419        for k, i in o.items():
420            st.append(k)
421            find_pickle_problem(i, st)
422            st.pop()
423    elif isinstance(o, list):
424        st.append('list')
425        for i in o:
426            find_pickle_problem(i, st)
427        st.pop()
428    else:
429        try:
430            pickle.dumps(o)
431        except pickle.PicklingError, e:
432            print >>sys.stderr, "<PicklingError>"
433            print >>sys.stderr, st
434            print >>sys.stderr, o
435            print >>sys.stderr, e
436            print >>sys.stderr, "</PicklingError>"
437
Note: See TracBrowser for help on using the repository browser.