source: fedd/federation/util.py @ 815cd26

compt_changes
Last change on this file since 815cd26 was 815cd26, checked in by Ted Faber <faber@…>, 12 years ago

Add timeout and extra debug logging.

  • Property mode set to 100644
File size: 12.3 KB
Line 
1#!/usr/local/bin/python
2
3import re
4import os
5import string
6import logging
7import pickle
8
9import httplib
10
11from optparse import OptionParser
12
13from socket import sslerror
14from tempfile import mkstemp
15
16from M2Crypto import SSL
17from M2Crypto.SSL import SSLError
18from deter import fedid
19from service_error import service_error
20from urlparse import urlparse
21from M2Crypto import m2
22
23
24# If this is an old enough version of M2Crypto.SSL that has an
25# ssl_verify_callback that doesn't allow 0-length signed certs, create a
26# version of that callback that does.  This is edited from the original in
27# M2Crypto.SSL.cb.  This version also elides the printing to stderr.
28if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None):
29    from M2Crypto.SSL.Context import map
30
31    def fedd_ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
32        unknown_issuer = [
33            m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY,
34            m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE,
35            m2.X509_V_ERR_CERT_UNTRUSTED,
36            m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
37            ]
38        # m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
39        if getattr(m2, 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None):
40            unknown_issuer.append(getattr(m2, 
41                'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None))
42        ssl_ctx = map()[ssl_ctx_ptr]
43
44        if errnum in unknown_issuer: 
45            if ssl_ctx.get_allow_unknown_ca():
46                ok = 1
47        # CRL checking goes here...
48        if ok:
49            if ssl_ctx.get_verify_depth() >= errdepth:
50                ok = 1
51            else:
52                ok = 0
53        return ok
54else:
55    def fedd_ssl_verify_callback(ok, store):
56        '''
57        m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
58        '''
59        errnum = store.get_error()
60        if errnum == m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
61            ok = 1
62            return ok
63        else:
64            return SSL.cb.ssl_verify_callback_allow_unknown_ca(ok, store)
65
66class fedd_ssl_context(SSL.Context):
67    """
68    Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd.
69    """
70    def __init__(self, my_cert, trusted_certs=None, password=None):
71        """
72        construct a fedd_ssl_context
73
74        @param my_cert: PEM file with my certificate in it
75        @param trusted_certs: PEM file with trusted certs in it (optional)
76        """
77        SSL.Context.__init__(self)
78
79        # load_cert takes a callback to get a password, not a password, so if
80        # the caller provided a password, this creates a nonce callback using a
81        # lambda form.
82        if password != None and not callable(password):
83            # This is cute.  password = lambda *args: password produces a
84            # function object that returns itself rather than one that returns
85            # the object itself.  This is because password is an object
86            # reference and after the assignment it's a lambda.  So we assign
87            # to a temp.
88            pwd = str(password)
89            password =lambda *args: pwd
90
91        # The calls to str below (and above) are because the underlying SSL
92        # stuff is intolerant of unicode.
93        if password != None:
94            self.load_cert(str(my_cert), callback=password)
95        else:
96            self.load_cert(str(my_cert))
97
98        # If no trusted certificates are specified, allow unknown CAs.
99        if trusted_certs: 
100            self.load_verify_locations(trusted_certs)
101            self.set_verify(SSL.verify_peer, 10)
102        else:
103            # Install the proper callback to allow self-signed certs
104            self.set_allow_unknown_ca(True)
105            self.set_verify(SSL.verify_peer, 10, 
106                    callback=fedd_ssl_verify_callback)
107
108        # no session caching
109        self.set_session_cache_mode(0)
110
111class file_expanding_opts(OptionParser):
112    def expand_file(self, option, opt_str, v, p):
113        """
114        Store the given value to the given destination after expanding home
115        directories.
116        """
117        setattr(p.values, option.dest, os.path.expanduser(v))
118
119    def __init__(self, usage=None, version=None):
120        OptionParser.__init__(self)
121
122
123def read_simple_accessdb(fn, auth, mask=[]):
124    """
125    Read a simple access database.  Each line is a fedid (of the form
126    fedid:hexstring) and a comma separated list of atributes to be assigned to
127    it.  This parses out the fedids and adds the attributes to the authorizer.
128    comments (preceded with a #) and blank lines are ignored.  Exceptions (e.g.
129    file exceptions and ValueErrors from badly parsed lines) are propagated.
130    """
131
132    rv = [ ]
133    lineno = 0
134    fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\
135            "(\w+\s*(,\s*\w+)*)")
136
137    # If a single string came in, make it a list
138    if isinstance(mask, basestring): mask = [ mask ]
139
140    f = open(fn, 'r')
141    for line in f:
142        lineno += 1
143        line = line.strip()
144        if line.startswith('#') or len(line) == 0: 
145            continue
146        m = fedid_line.match(line)
147        if m :
148            fid = fedid(hexstr=m.group(1))
149            for a in [ a.strip() for a in m.group(2).split(",") \
150                    if not mask or a.strip() in mask ]:
151                auth.set_attribute(fid, a.strip())
152        else:
153            raise ValueError("Badly formatted line in accessdb: %s line %d" %\
154                    (fn, lineno))
155    f.close()
156    return rv
157       
158
159def pack_id(id):
160    """
161    Return a dictionary with the field name set by the id type.  Handy for
162    creating dictionaries to be converted to messages.
163    """
164    if isinstance(id, fedid): return { 'fedid': id }
165    elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id }
166    else: return { 'localname': id}
167
168def unpack_id(id):
169    """return id as a type determined by the key"""
170    for k in ("localname", "fedid", "uri", "kerberosUsername"):
171        if id.has_key(k): return id[k]
172    return None
173
174def set_log_level(config, sect, log):
175    """ Set the logging level to the value passed in sect of config."""
176    # The getattr sleight of hand finds the logging level constant
177    # corrersponding to the string.  We're a little paranoid to avoid user
178    # mayhem.
179    if config.has_option(sect, "log_level"):
180        level_str = config.get(sect, "log_level")
181        try:
182            level = int(getattr(logging, level_str.upper(), -1))
183
184            if  logging.DEBUG <= level <= logging.CRITICAL:
185                log.setLevel(level)
186            else:
187                log.error("Bad experiment_log value: %s" % level_str)
188
189        except ValueError:
190            log.error("Bad experiment_log value: %s" % level_str)
191
192def copy_file(src, dest, size=1024):
193    """
194    Exceedingly simple file copy.  Throws an EnvironmentError if there's a problem.
195    """
196    s = open(src,'r')
197    d = open(dest, 'w')
198
199    buf = s.read(size)
200    while buf != "":
201        d.write(buf)
202        buf = s.read(size)
203    s.close()
204    d.close()
205
206def get_url(url, cf, destdir, fn=None, max_retries=5, log=None):
207    """
208    Get data from a federated data store.  This presents the client cert/fedid
209    to the http server.  We retry up to max_retries times.
210    """
211    po = urlparse(url)
212    if not fn:
213        fn = po.path.rpartition('/')[2]
214    retries = 0
215    ok = False
216    failed_exception = None
217    while not ok and retries < 5:
218        try:
219            if log:
220                log.debug('Creating HTTPSConnection')
221            conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
222                    cert_file=cf, key_file=cf, timeout=30)
223            conn.putrequest('GET', po.path)
224            conn.endheaders()
225            if log:
226                log.debug('Connecting')
227            response = conn.getresponse()
228            if log:
229                log.debug('Connected')
230
231            lf = open("%s/%s" % (destdir, fn), "w")
232            if log:
233                log.debug('initial read')
234            buf = response.read(4096)
235            while buf:
236                lf.write(buf)
237                if log:
238                    log.debug('loop read')
239                buf = response.read(4096)
240            lf.close()
241            ok = True
242        except EnvironmentError, e:
243            failed_excpetion = e
244            retries += 1
245        except httplib.HTTPException, e:
246            failed_exception = e
247            retries += 1
248        except sslerror, e:
249            failed_exception = e
250            retries += 1
251        except SSLError, e:
252            failed_exception = e
253            retries += 1
254
255    if retries > 5 and failed_exception:
256        raise failed_excpetion
257
258# Functions to manipulate composite testbed names
259def testbed_base(tb):
260    """
261    Simple code to get the base testebd name.
262    """
263    i = tb.find('/')
264    if i == -1: return tb
265    else: return tb[0:i]
266
267def testbed_suffix(tb):
268    """
269    Simple code to get a testbed suffix, if nay.  No suffix returns None.
270    """
271    i = tb.find('/')
272    if i != -1: return tb[i+1:]
273    else: return None
274
275def split_testbed(tb):
276    """
277    Return a testbed and a suffix as a tuple.  No suffix returns None for that
278    field
279    """
280
281    i = tb.find('/')
282    if i != -1: return (tb[0:i], tb[i+1:])
283    else: return (tb, None)
284
285def join_testbed(base, suffix=None):
286    """
287    Build a testbed with suffix.  If base is itself a tuple, combine them,
288    otherwise combine the two.
289    """
290    if isinstance(base, tuple):
291        if len(base) == 2:
292            return '/'.join(base)
293        else:
294            raise RuntimeError("Too many tuple elements for join_testbed")
295    else:
296        if suffix:
297            return '/'.join((base, suffix))
298        else:
299            return base
300
301def abac_pem_type(cert):
302    key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$')
303    cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') 
304    type = None
305
306    f = open(cert, 'r')
307    for line in f:
308        if key_re.match(line):
309            if type is None: type = 'key'
310            elif type == 'cert': type = 'both'
311        elif cert_re.match(line):
312            if type is None: type = 'cert'
313            elif type == 'key': type = 'both'
314        if type == 'both': break
315    f.close()
316    return type
317
318def abac_split_cert(cert, keyfile=None, certfile=None):
319    """
320    Split the certificate file in cert into a certificate file and a key file
321    in cf and kf respectively.  The ABAC tools generally cannot handle combined
322    certificates/keys.  If kf anc cf are given, they are used, otherwise tmp
323    files are created.  Created tmp files must be deleted.  Problems opening or
324    writing files will cause exceptions.
325    """
326    class diversion:
327        '''
328        Wraps up the reqular expression to start and end a diversion, as well as
329        the open file that gets the lines.  If fd is passed in, use that system
330        file (probably from a mkstemp.  Otherwise open the given filename.
331        '''
332        def __init__(self, start, end, fn=None, fd=None):
333            self.start = re.compile(start)
334            self.end = re.compile(end)
335
336            if not fd:
337                # Open the file securely with minimal permissions. NB file
338                # cannot exist before this call.
339                fd = os.open(fn,
340                    (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600)
341
342            self.f = os.fdopen(fd, 'w')
343
344        def close(self):
345            self.f.close()
346
347    if not keyfile:
348        kf, rkeyfile = mkstemp(suffix=".pem")
349    else:
350        kf, rkeyfile = None, keyfile
351
352    if not certfile:
353        cf, rcertfile = mkstemp(suffix=".pem")
354    else:
355        cf, rcertfile = None, certfile
356
357    # Initialize the diversions
358    divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in (
359        ('\s*-----BEGIN RSA PRIVATE KEY-----$', 
360            '\s*-----END RSA PRIVATE KEY-----$',
361            keyfile, kf),
362        ('\s*-----BEGIN CERTIFICATE-----$', 
363            '\s*-----END CERTIFICATE-----$',
364            certfile, cf))]
365
366    # walk through the file, beginning a diversion when a start regexp
367    # matches until the end regexp matches.  While in the two regexps,
368    # print each line to the open diversion file (including the two
369    # matches).
370    active = None
371    f = open(cert, 'r')
372    for l in f:
373        if active:
374            if active.end.match(l):
375                print >>active.f, l,
376                active = None
377        else:
378            for d in divs:
379                if d.start.match(l):
380                    active = d
381                    break
382        if active: print >>active.f, l,
383
384    # This is probably unnecessary.  Close all the diversion files.
385    for d in divs: d.close()
386    return rkeyfile, rcertfile
387
388def abac_context_to_creds(context):
389    """
390    Pull all the credentials out of the context and return 2  lists of the
391    underlying credentials in an exportable format, IDs and attributes.
392    There are no duplicates in the lists.
393    """
394    ids, attrs = set(), set()
395    # This should be a one-iteration loop
396    for c in context.credentials():
397        ids.add(str(c.issuer_cert()))
398        attrs.add(str(c.attribute_cert()))
399
400    return list(ids), list(attrs)
401
402def find_pickle_problem(o, st=None):
403    """
404    Debugging routine to figure out what doesn't pickle from a dict full of
405    dicts and lists.  It tries to walk down the lists and dicts and pickle each
406    atom.  If something fails to pickle, it prints an approximation of a stack
407    trace through the data structure.
408    """
409    if st is None: st = [ ]
410
411    if isinstance(o, dict):
412        for k, i in o.items():
413            st.append(k)
414            find_pickle_problem(i, st)
415            st.pop()
416    elif isinstance(o, list):
417        st.append('list')
418        for i in o:
419            find_pickle_problem(i, st)
420        st.pop()
421    else:
422        try:
423            pickle.dumps(o)
424        except pickle.PicklingError, e:
425            print >>sys.stderr, "<PicklingError>"
426            print >>sys.stderr, st
427            print >>sys.stderr, o
428            print >>sys.stderr, e
429            print >>sys.stderr, "</PicklingError>"
430
Note: See TracBrowser for help on using the repository browser.