source: fedd/federation/util.py @ 7653f01

compt_changes
Last change on this file since 7653f01 was 6bedbdba, checked in by Ted Faber <faber@…>, 13 years ago

Split topdl and fedid out to different packages. Add differential
installs

  • Property mode set to 100644
File size: 12.1 KB
Line 
1#!/usr/local/bin/python
2
3import re
4import os
5import string
6import logging
7import pickle
8
9import httplib
10
11from optparse import OptionParser
12
13from socket import sslerror
14from tempfile import mkstemp
15
16from M2Crypto import SSL
17from M2Crypto.SSL import SSLError
18from deter import fedid
19from service_error import service_error
20from urlparse import urlparse
21from M2Crypto import m2
22
23
24# If this is an old enough version of M2Crypto.SSL that has an
25# ssl_verify_callback that doesn't allow 0-length signed certs, create a
26# version of that callback that does.  This is edited from the original in
27# M2Crypto.SSL.cb.  This version also elides the printing to stderr.
28if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None):
29    from M2Crypto.SSL.Context import map
30
31    def fedd_ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
32        unknown_issuer = [
33            m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY,
34            m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE,
35            m2.X509_V_ERR_CERT_UNTRUSTED,
36            m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
37            ]
38        # m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
39        if getattr(m2, 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None):
40            unknown_issuer.append(getattr(m2, 
41                'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None))
42        ssl_ctx = map()[ssl_ctx_ptr]
43
44        if errnum in unknown_issuer: 
45            if ssl_ctx.get_allow_unknown_ca():
46                ok = 1
47        # CRL checking goes here...
48        if ok:
49            if ssl_ctx.get_verify_depth() >= errdepth:
50                ok = 1
51            else:
52                ok = 0
53        return ok
54else:
55    def fedd_ssl_verify_callback(ok, store):
56        '''
57        m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed
58        '''
59        errnum = store.get_error()
60        if errnum == m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
61            ok = 1
62            return ok
63        else:
64            return SSL.cb.ssl_verify_callback_allow_unknown_ca(ok, store)
65
66class fedd_ssl_context(SSL.Context):
67    """
68    Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd.
69    """
70    def __init__(self, my_cert, trusted_certs=None, password=None):
71        """
72        construct a fedd_ssl_context
73
74        @param my_cert: PEM file with my certificate in it
75        @param trusted_certs: PEM file with trusted certs in it (optional)
76        """
77        SSL.Context.__init__(self)
78
79        # load_cert takes a callback to get a password, not a password, so if
80        # the caller provided a password, this creates a nonce callback using a
81        # lambda form.
82        if password != None and not callable(password):
83            # This is cute.  password = lambda *args: password produces a
84            # function object that returns itself rather than one that returns
85            # the object itself.  This is because password is an object
86            # reference and after the assignment it's a lambda.  So we assign
87            # to a temp.
88            pwd = str(password)
89            password =lambda *args: pwd
90
91        # The calls to str below (and above) are because the underlying SSL
92        # stuff is intolerant of unicode.
93        if password != None:
94            self.load_cert(str(my_cert), callback=password)
95        else:
96            self.load_cert(str(my_cert))
97
98        # If no trusted certificates are specified, allow unknown CAs.
99        if trusted_certs: 
100            self.load_verify_locations(trusted_certs)
101            self.set_verify(SSL.verify_peer, 10)
102        else:
103            # Install the proper callback to allow self-signed certs
104            self.set_allow_unknown_ca(True)
105            self.set_verify(SSL.verify_peer, 10, 
106                    callback=fedd_ssl_verify_callback)
107
108        # no session caching
109        self.set_session_cache_mode(0)
110
111class file_expanding_opts(OptionParser):
112    def expand_file(self, option, opt_str, v, p):
113        """
114        Store the given value to the given destination after expanding home
115        directories.
116        """
117        setattr(p.values, option.dest, os.path.expanduser(v))
118
119    def __init__(self, usage=None, version=None):
120        OptionParser.__init__(self)
121
122
123def read_simple_accessdb(fn, auth, mask=[]):
124    """
125    Read a simple access database.  Each line is a fedid (of the form
126    fedid:hexstring) and a comma separated list of atributes to be assigned to
127    it.  This parses out the fedids and adds the attributes to the authorizer.
128    comments (preceded with a #) and blank lines are ignored.  Exceptions (e.g.
129    file exceptions and ValueErrors from badly parsed lines) are propagated.
130    """
131
132    rv = [ ]
133    lineno = 0
134    fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\
135            "(\w+\s*(,\s*\w+)*)")
136
137    # If a single string came in, make it a list
138    if isinstance(mask, basestring): mask = [ mask ]
139
140    f = open(fn, 'r')
141    for line in f:
142        lineno += 1
143        line = line.strip()
144        if line.startswith('#') or len(line) == 0: 
145            continue
146        m = fedid_line.match(line)
147        if m :
148            fid = fedid(hexstr=m.group(1))
149            for a in [ a.strip() for a in m.group(2).split(",") \
150                    if not mask or a.strip() in mask ]:
151                auth.set_attribute(fid, a.strip())
152        else:
153            raise ValueError("Badly formatted line in accessdb: %s line %d" %\
154                    (fn, lineno))
155    f.close()
156    return rv
157       
158
159def pack_id(id):
160    """
161    Return a dictionary with the field name set by the id type.  Handy for
162    creating dictionaries to be converted to messages.
163    """
164    if isinstance(id, fedid): return { 'fedid': id }
165    elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id }
166    else: return { 'localname': id}
167
168def unpack_id(id):
169    """return id as a type determined by the key"""
170    for k in ("localname", "fedid", "uri", "kerberosUsername"):
171        if id.has_key(k): return id[k]
172    return None
173
174def set_log_level(config, sect, log):
175    """ Set the logging level to the value passed in sect of config."""
176    # The getattr sleight of hand finds the logging level constant
177    # corrersponding to the string.  We're a little paranoid to avoid user
178    # mayhem.
179    if config.has_option(sect, "log_level"):
180        level_str = config.get(sect, "log_level")
181        try:
182            level = int(getattr(logging, level_str.upper(), -1))
183
184            if  logging.DEBUG <= level <= logging.CRITICAL:
185                log.setLevel(level)
186            else:
187                log.error("Bad experiment_log value: %s" % level_str)
188
189        except ValueError:
190            log.error("Bad experiment_log value: %s" % level_str)
191
192def copy_file(src, dest, size=1024):
193    """
194    Exceedingly simple file copy.  Throws an EnvironmentError if there's a problem.
195    """
196    s = open(src,'r')
197    d = open(dest, 'w')
198
199    buf = s.read(size)
200    while buf != "":
201        d.write(buf)
202        buf = s.read(size)
203    s.close()
204    d.close()
205
206def get_url(url, cf, destdir, fn=None, max_retries=5):
207    """
208    Get data from a federated data store.  This presents the client cert/fedid
209    to the http server.  We retry up to max_retries times.
210    """
211    po = urlparse(url)
212    if not fn:
213        fn = po.path.rpartition('/')[2]
214    retries = 0
215    ok = False
216    failed_exception = None
217    while not ok and retries < 5:
218        try:
219            conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
220                    cert_file=cf, key_file=cf)
221            conn.putrequest('GET', po.path)
222            conn.endheaders()
223            response = conn.getresponse()
224
225            lf = open("%s/%s" % (destdir, fn), "w")
226            buf = response.read(4096)
227            while buf:
228                lf.write(buf)
229                buf = response.read(4096)
230            lf.close()
231            ok = True
232        except EnvironmentError, e:
233            failed_excpetion = e
234            retries += 1
235        except httplib.HTTPException, e:
236            failed_exception = e
237            retries += 1
238        except sslerror, e:
239            failed_exception = e
240            retries += 1
241        except SSLError, e:
242            failed_exception = e
243            retries += 1
244
245    if retries > 5 and failed_exception:
246        raise failed_excpetion
247
248# Functions to manipulate composite testbed names
249def testbed_base(tb):
250    """
251    Simple code to get the base testebd name.
252    """
253    i = tb.find('/')
254    if i == -1: return tb
255    else: return tb[0:i]
256
257def testbed_suffix(tb):
258    """
259    Simple code to get a testbed suffix, if nay.  No suffix returns None.
260    """
261    i = tb.find('/')
262    if i != -1: return tb[i+1:]
263    else: return None
264
265def split_testbed(tb):
266    """
267    Return a testbed and a suffix as a tuple.  No suffix returns None for that
268    field
269    """
270
271    i = tb.find('/')
272    if i != -1: return (tb[0:i], tb[i+1:])
273    else: return (tb, None)
274
275def join_testbed(base, suffix=None):
276    """
277    Build a testbed with suffix.  If base is itself a tuple, combine them,
278    otherwise combine the two.
279    """
280    if isinstance(base, tuple):
281        if len(base) == 2:
282            return '/'.join(base)
283        else:
284            raise RuntimeError("Too many tuple elements for join_testbed")
285    else:
286        if suffix:
287            return '/'.join((base, suffix))
288        else:
289            return base
290
291def abac_pem_type(cert):
292    key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$')
293    cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') 
294    type = None
295
296    f = open(cert, 'r')
297    for line in f:
298        if key_re.match(line):
299            if type is None: type = 'key'
300            elif type == 'cert': type = 'both'
301        elif cert_re.match(line):
302            if type is None: type = 'cert'
303            elif type == 'key': type = 'both'
304        if type == 'both': break
305    f.close()
306    return type
307
308def abac_split_cert(cert, keyfile=None, certfile=None):
309    """
310    Split the certificate file in cert into a certificate file and a key file
311    in cf and kf respectively.  The ABAC tools generally cannot handle combined
312    certificates/keys.  If kf anc cf are given, they are used, otherwise tmp
313    files are created.  Created tmp files must be deleted.  Problems opening or
314    writing files will cause exceptions.
315    """
316    class diversion:
317        '''
318        Wraps up the reqular expression to start and end a diversion, as well as
319        the open file that gets the lines.  If fd is passed in, use that system
320        file (probably from a mkstemp.  Otherwise open the given filename.
321        '''
322        def __init__(self, start, end, fn=None, fd=None):
323            self.start = re.compile(start)
324            self.end = re.compile(end)
325
326            if not fd:
327                # Open the file securely with minimal permissions. NB file
328                # cannot exist before this call.
329                fd = os.open(fn,
330                    (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600)
331
332            self.f = os.fdopen(fd, 'w')
333
334        def close(self):
335            self.f.close()
336
337    if not keyfile:
338        kf, rkeyfile = mkstemp(suffix=".pem")
339    else:
340        kf, rkeyfile = None, keyfile
341
342    if not certfile:
343        cf, rcertfile = mkstemp(suffix=".pem")
344    else:
345        cf, rcertfile = None, certfile
346
347    # Initialize the diversions
348    divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in (
349        ('\s*-----BEGIN RSA PRIVATE KEY-----$', 
350            '\s*-----END RSA PRIVATE KEY-----$',
351            keyfile, kf),
352        ('\s*-----BEGIN CERTIFICATE-----$', 
353            '\s*-----END CERTIFICATE-----$',
354            certfile, cf))]
355
356    # walk through the file, beginning a diversion when a start regexp
357    # matches until the end regexp matches.  While in the two regexps,
358    # print each line to the open diversion file (including the two
359    # matches).
360    active = None
361    f = open(cert, 'r')
362    for l in f:
363        if active:
364            if active.end.match(l):
365                print >>active.f, l,
366                active = None
367        else:
368            for d in divs:
369                if d.start.match(l):
370                    active = d
371                    break
372        if active: print >>active.f, l,
373
374    # This is probably unnecessary.  Close all the diversion files.
375    for d in divs: d.close()
376    return rkeyfile, rcertfile
377
378def abac_context_to_creds(context):
379    """
380    Pull all the credentials out of the context and return 2  lists of the
381    underlying credentials in an exportable format, IDs and attributes.
382    There are no duplicates in the lists.
383    """
384    ids, attrs = set(), set()
385    # This should be a one-iteration loop
386    for c in context.credentials():
387        ids.add(str(c.issuer_cert()))
388        attrs.add(str(c.attribute_cert()))
389
390    return list(ids), list(attrs)
391
392def find_pickle_problem(o, st=None):
393    """
394    Debugging routine to figure out what doesn't pickle from a dict full of
395    dicts and lists.  It tries to walk down the lists and dicts and pickle each
396    atom.  If something fails to pickle, it prints an approximation of a stack
397    trace through the data structure.
398    """
399    if st is None: st = [ ]
400
401    if isinstance(o, dict):
402        for k, i in o.items():
403            st.append(k)
404            find_pickle_problem(i, st)
405            st.pop()
406    elif isinstance(o, list):
407        st.append('list')
408        for i in o:
409            find_pickle_problem(i, st)
410        st.pop()
411    else:
412        try:
413            pickle.dumps(o)
414        except pickle.PicklingError, e:
415            print >>sys.stderr, "<PicklingError>"
416            print >>sys.stderr, st
417            print >>sys.stderr, o
418            print >>sys.stderr, e
419            print >>sys.stderr, "</PicklingError>"
420
Note: See TracBrowser for help on using the repository browser.