source: fedd/federation/util.py @ d070d9f

Last change on this file since d070d9f was f490c75, checked in by Ted Faber <faber@…>, 12 years ago

Use the max_retries parameter (!?)

  • Property mode set to 100644
File size: 12.4 KB
Line 
1#!/usr/local/bin/python
2
3import re
4import os
5import string
6import logging
7import pickle
8
9import httplib
10
11from optparse import OptionParser
12
13from socket import sslerror
14from tempfile import mkstemp
15
16from M2Crypto import SSL
17from M2Crypto.SSL import SSLError
18from deter import fedid
19from service_error import service_error
20from urlparse import urlparse
21from M2Crypto import m2
22
23
24# If this is an old enough version of M2Crypto.SSL that has an
25# ssl_verify_callback that doesn't allow 0-length signed certs, create a
26# version of that callback that does.  This is edited from the original in
27# M2Crypto.SSL.cb.  This version also elides the printing to stderr.
28if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None):
29    from M2Crypto.SSL.Context import map
30
31    def fedd_ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
32        unknown_issuer = [
33            m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY,
34            m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE,
35            m2.X509_V_ERR_CERT_UNTRUSTED,
36            m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
37            ]
38        # m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
39        if getattr(m2, 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None):
40            unknown_issuer.append(getattr(m2, 
41                'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None))
42        ssl_ctx = map()[ssl_ctx_ptr]
43
44        if errnum in unknown_issuer: 
45            if ssl_ctx.get_allow_unknown_ca():
46                ok = 1
47        # CRL checking goes here...
48        if ok:
49            if ssl_ctx.get_verify_depth() >= errdepth:
50                ok = 1
51            else:
52                ok = 0
53        return ok
54else:
55    def fedd_ssl_verify_callback(ok, store):
56        '''
57        m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
58        '''
59        errnum = store.get_error()
60        if errnum == m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
61            ok = 1
62            return ok
63        else:
64            return SSL.cb.ssl_verify_callback_allow_unknown_ca(ok, store)
65
66class fedd_ssl_context(SSL.Context):
67    """
68    Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd.
69    """
70    def __init__(self, my_cert, trusted_certs=None, password=None):
71        """
72        construct a fedd_ssl_context
73
74        @param my_cert: PEM file with my certificate in it
75        @param trusted_certs: PEM file with trusted certs in it (optional)
76        """
77        SSL.Context.__init__(self)
78
79        # load_cert takes a callback to get a password, not a password, so if
80        # the caller provided a password, this creates a nonce callback using a
81        # lambda form.
82        if password != None and not callable(password):
83            # This is cute.  password = lambda *args: password produces a
84            # function object that returns itself rather than one that returns
85            # the object itself.  This is because password is an object
86            # reference and after the assignment it's a lambda.  So we assign
87            # to a temp.
88            pwd = str(password)
89            password =lambda *args: pwd
90
91        # The calls to str below (and above) are because the underlying SSL
92        # stuff is intolerant of unicode.
93        if password != None:
94            self.load_cert(str(my_cert), callback=password)
95        else:
96            self.load_cert(str(my_cert))
97
98        # If no trusted certificates are specified, allow unknown CAs.
99        if trusted_certs: 
100            self.load_verify_locations(trusted_certs)
101            self.set_verify(SSL.verify_peer, 10)
102        else:
103            # Install the proper callback to allow self-signed certs
104            self.set_allow_unknown_ca(True)
105            self.set_verify(SSL.verify_peer, 10, 
106                    callback=fedd_ssl_verify_callback)
107
108        # no session caching
109        self.set_session_cache_mode(0)
110
111class file_expanding_opts(OptionParser):
112    def expand_file(self, option, opt_str, v, p):
113        """
114        Store the given value to the given destination after expanding home
115        directories.
116        """
117        setattr(p.values, option.dest, os.path.expanduser(v))
118
119    def __init__(self, usage=None, version=None):
120        OptionParser.__init__(self)
121
122
123def read_simple_accessdb(fn, auth, mask=[]):
124    """
125    Read a simple access database.  Each line is a fedid (of the form
126    fedid:hexstring) and a comma separated list of atributes to be assigned to
127    it.  This parses out the fedids and adds the attributes to the authorizer.
128    comments (preceded with a #) and blank lines are ignored.  Exceptions (e.g.
129    file exceptions and ValueErrors from badly parsed lines) are propagated.
130    """
131
132    rv = [ ]
133    lineno = 0
134    fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\
135            "(\w+\s*(,\s*\w+)*)")
136
137    # If a single string came in, make it a list
138    if isinstance(mask, basestring): mask = [ mask ]
139
140    f = open(fn, 'r')
141    for line in f:
142        lineno += 1
143        line = line.strip()
144        if line.startswith('#') or len(line) == 0: 
145            continue
146        m = fedid_line.match(line)
147        if m :
148            fid = fedid(hexstr=m.group(1))
149            for a in [ a.strip() for a in m.group(2).split(",") \
150                    if not mask or a.strip() in mask ]:
151                auth.set_attribute(fid, a.strip())
152        else:
153            raise ValueError("Badly formatted line in accessdb: %s line %d" %\
154                    (fn, lineno))
155    f.close()
156    return rv
157       
158
159def pack_id(id):
160    """
161    Return a dictionary with the field name set by the id type.  Handy for
162    creating dictionaries to be converted to messages.
163    """
164    if isinstance(id, fedid): return { 'fedid': id }
165    elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id }
166    else: return { 'localname': id}
167
168def unpack_id(id):
169    """return id as a type determined by the key"""
170    for k in ("localname", "fedid", "uri", "kerberosUsername"):
171        if id.has_key(k): return id[k]
172    return None
173
174def set_log_level(config, sect, log):
175    """ Set the logging level to the value passed in sect of config."""
176    # The getattr sleight of hand finds the logging level constant
177    # corrersponding to the string.  We're a little paranoid to avoid user
178    # mayhem.
179    if config.has_option(sect, "log_level"):
180        level_str = config.get(sect, "log_level")
181        try:
182            level = int(getattr(logging, level_str.upper(), -1))
183
184            if  logging.DEBUG <= level <= logging.CRITICAL:
185                log.setLevel(level)
186            else:
187                log.error("Bad experiment_log value: %s" % level_str)
188
189        except ValueError:
190            log.error("Bad experiment_log value: %s" % level_str)
191
192def copy_file(src, dest, size=1024):
193    """
194    Exceedingly simple file copy.  Throws an EnvironmentError if there's a problem.
195    """
196    s = open(src,'r')
197    d = open(dest, 'w')
198
199    buf = s.read(size)
200    while buf != "":
201        d.write(buf)
202        buf = s.read(size)
203    s.close()
204    d.close()
205
206def get_url(url, cf, destdir, fn=None, max_retries=5, log=None):
207    """
208    Get data from a federated data store.  This presents the client cert/fedid
209    to the http server.  We retry up to max_retries times.
210    """
211    po = urlparse(url)
212    if not fn:
213        fn = po.path.rpartition('/')[2]
214    retries = 0
215    ok = False
216    failed_exception = None
217    while not ok and retries < max_retries:
218        try:
219            if log:
220                log.debug('Creating HTTPSConnection: %s' % cf)
221            conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
222                    cert_file=cf, key_file=cf, timeout=30)
223            conn.putrequest('GET', po.path)
224            conn.endheaders()
225            if log:
226                log.debug('Connecting')
227            response = conn.getresponse()
228            if log:
229                log.debug('Connected')
230
231            lf = open("%s/%s" % (destdir, fn), "w")
232            if log:
233                log.debug('initial read')
234            buf = response.read(4096)
235            while buf:
236                lf.write(buf)
237                if log:
238                    log.debug('loop read')
239                buf = response.read(4096)
240            lf.close()
241            ok = True
242        except EnvironmentError, e:
243            failed_exception = e
244            retries += 1
245        except httplib.HTTPException, e:
246            failed_exception = e
247            retries += 1
248        except sslerror, e:
249            failed_exception = e
250            retries += 1
251        except SSLError, e:
252            failed_exception = e
253            retries += 1
254
255    if retries > max_retries and failed_exception:
256        if log:
257            log.debug('Raising %s', failed_exception)
258        raise failed_excpetion
259
260# Functions to manipulate composite testbed names
261def testbed_base(tb):
262    """
263    Simple code to get the base testebd name.
264    """
265    i = tb.find('/')
266    if i == -1: return tb
267    else: return tb[0:i]
268
269def testbed_suffix(tb):
270    """
271    Simple code to get a testbed suffix, if nay.  No suffix returns None.
272    """
273    i = tb.find('/')
274    if i != -1: return tb[i+1:]
275    else: return None
276
277def split_testbed(tb):
278    """
279    Return a testbed and a suffix as a tuple.  No suffix returns None for that
280    field
281    """
282
283    i = tb.find('/')
284    if i != -1: return (tb[0:i], tb[i+1:])
285    else: return (tb, None)
286
287def join_testbed(base, suffix=None):
288    """
289    Build a testbed with suffix.  If base is itself a tuple, combine them,
290    otherwise combine the two.
291    """
292    if isinstance(base, tuple):
293        if len(base) == 2:
294            return '/'.join(base)
295        else:
296            raise RuntimeError("Too many tuple elements for join_testbed")
297    else:
298        if suffix:
299            return '/'.join((base, suffix))
300        else:
301            return base
302
303def abac_pem_type(cert):
304    key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$')
305    cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') 
306    type = None
307
308    f = open(cert, 'r')
309    for line in f:
310        if key_re.match(line):
311            if type is None: type = 'key'
312            elif type == 'cert': type = 'both'
313        elif cert_re.match(line):
314            if type is None: type = 'cert'
315            elif type == 'key': type = 'both'
316        if type == 'both': break
317    f.close()
318    return type
319
320def abac_split_cert(cert, keyfile=None, certfile=None):
321    """
322    Split the certificate file in cert into a certificate file and a key file
323    in cf and kf respectively.  The ABAC tools generally cannot handle combined
324    certificates/keys.  If kf anc cf are given, they are used, otherwise tmp
325    files are created.  Created tmp files must be deleted.  Problems opening or
326    writing files will cause exceptions.
327    """
328    class diversion:
329        '''
330        Wraps up the reqular expression to start and end a diversion, as well as
331        the open file that gets the lines.  If fd is passed in, use that system
332        file (probably from a mkstemp.  Otherwise open the given filename.
333        '''
334        def __init__(self, start, end, fn=None, fd=None):
335            self.start = re.compile(start)
336            self.end = re.compile(end)
337
338            if not fd:
339                # Open the file securely with minimal permissions. NB file
340                # cannot exist before this call.
341                fd = os.open(fn,
342                    (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600)
343
344            self.f = os.fdopen(fd, 'w')
345
346        def close(self):
347            self.f.close()
348
349    if not keyfile:
350        kf, rkeyfile = mkstemp(suffix=".pem")
351    else:
352        kf, rkeyfile = None, keyfile
353
354    if not certfile:
355        cf, rcertfile = mkstemp(suffix=".pem")
356    else:
357        cf, rcertfile = None, certfile
358
359    # Initialize the diversions
360    divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in (
361        ('\s*-----BEGIN RSA PRIVATE KEY-----$', 
362            '\s*-----END RSA PRIVATE KEY-----$',
363            keyfile, kf),
364        ('\s*-----BEGIN CERTIFICATE-----$', 
365            '\s*-----END CERTIFICATE-----$',
366            certfile, cf))]
367
368    # walk through the file, beginning a diversion when a start regexp
369    # matches until the end regexp matches.  While in the two regexps,
370    # print each line to the open diversion file (including the two
371    # matches).
372    active = None
373    f = open(cert, 'r')
374    for l in f:
375        if active:
376            if active.end.match(l):
377                print >>active.f, l,
378                active = None
379        else:
380            for d in divs:
381                if d.start.match(l):
382                    active = d
383                    break
384        if active: print >>active.f, l,
385
386    # This is probably unnecessary.  Close all the diversion files.
387    for d in divs: d.close()
388    return rkeyfile, rcertfile
389
390def abac_context_to_creds(context):
391    """
392    Pull all the credentials out of the context and return 2  lists of the
393    underlying credentials in an exportable format, IDs and attributes.
394    There are no duplicates in the lists.
395    """
396    ids, attrs = set(), set()
397    # This should be a one-iteration loop
398    for c in context.credentials():
399        ids.add(str(c.issuer_cert()))
400        attrs.add(str(c.attribute_cert()))
401
402    return list(ids), list(attrs)
403
404def find_pickle_problem(o, st=None):
405    """
406    Debugging routine to figure out what doesn't pickle from a dict full of
407    dicts and lists.  It tries to walk down the lists and dicts and pickle each
408    atom.  If something fails to pickle, it prints an approximation of a stack
409    trace through the data structure.
410    """
411    if st is None: st = [ ]
412
413    if isinstance(o, dict):
414        for k, i in o.items():
415            st.append(k)
416            find_pickle_problem(i, st)
417            st.pop()
418    elif isinstance(o, list):
419        st.append('list')
420        for i in o:
421            find_pickle_problem(i, st)
422        st.pop()
423    else:
424        try:
425            pickle.dumps(o)
426        except pickle.PicklingError, e:
427            print >>sys.stderr, "<PicklingError>"
428            print >>sys.stderr, st
429            print >>sys.stderr, o
430            print >>sys.stderr, e
431            print >>sys.stderr, "</PicklingError>"
432
Note: See TracBrowser for help on using the repository browser.