source: fedd/federation/util.py @ 62f3dd9

axis_examplecompt_changesinfo-ops
Last change on this file since 62f3dd9 was 62f3dd9, checked in by Ted Faber <faber@…>, 13 years ago

allow command line progams to expand tildes. Added a class derived from OptionParser? to make that easily available.

  • Property mode set to 100644
File size: 11.9 KB
Line 
1#!/usr/local/bin/python
2
3import re
4import os
5import string
6import logging
7import pickle
8
9import httplib
10
11from optparse import OptionParser
12
13from socket import sslerror
14from tempfile import mkstemp
15
16from M2Crypto import SSL
17from M2Crypto.SSL import SSLError
18from fedid import fedid
19from service_error import service_error
20from urlparse import urlparse
21
22
23# If this is an old enough version of M2Crypto.SSL that has an
24# ssl_verify_callback that doesn't allow 0-length signed certs, create a
25# version of that callback that does.  This is edited from the original in
26# M2Crypto.SSL.cb.  This version also elides the printing to stderr.
27if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None):
28    from M2Crypto.SSL.Context import map
29    from M2Crypto import m2
30
31    def ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
32        unknown_issuer = [
33            m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY,
34            m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE,
35            m2.X509_V_ERR_CERT_UNTRUSTED,
36            m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
37            ]
38        ssl_ctx = map()[ssl_ctx_ptr]
39
40        if errnum in unknown_issuer: 
41            if ssl_ctx.get_allow_unknown_ca():
42                ok = 1
43        # CRL checking goes here...
44        if ok:
45            if ssl_ctx.get_verify_depth() >= errdepth:
46                ok = 1
47            else:
48                ok = 0
49        return ok
50else:
51    def ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
52        raise ValueError("This should never be called")
53
54class fedd_ssl_context(SSL.Context):
55    """
56    Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd.
57    """
58    def __init__(self, my_cert, trusted_certs=None, password=None):
59        """
60        construct a fedd_ssl_context
61
62        @param my_cert: PEM file with my certificate in it
63        @param trusted_certs: PEM file with trusted certs in it (optional)
64        """
65        SSL.Context.__init__(self)
66
67        # load_cert takes a callback to get a password, not a password, so if
68        # the caller provided a password, this creates a nonce callback using a
69        # lambda form.
70        if password != None and not callable(password):
71            # This is cute.  password = lambda *args: password produces a
72            # function object that returns itself rather than one that returns
73            # the object itself.  This is because password is an object
74            # reference and after the assignment it's a lambda.  So we assign
75            # to a temp.
76            pwd = str(password)
77            password =lambda *args: pwd
78
79        # The calls to str below (and above) are because the underlying SSL
80        # stuff is intolerant of unicode.
81        if password != None:
82            self.load_cert(str(my_cert), callback=password)
83        else:
84            self.load_cert(str(my_cert))
85
86        # If no trusted certificates are specified, allow unknown CAs.
87        if trusted_certs: 
88            self.load_verify_locations(trusted_certs)
89            self.set_verify(SSL.verify_peer, 10)
90        else:
91            # More legacy code.  Recent versions of M2Crypto express the
92            # allow_unknown_ca option through a callback turned to allow it.
93            # Older versions use a standard callback that respects the
94            # attribute.  This should work under both regines.
95            callb = getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', 
96                    ssl_verify_callback)
97            self.set_allow_unknown_ca(True)
98            self.set_verify(SSL.verify_peer, 10, callback=callb)
99
100class file_expanding_opts(OptionParser):
101    def expand_file(self, option, opt_str, v, p):
102        """
103        Store the given value to the given destination after expanding home
104        directories.
105        """
106        setattr(p.values, option.dest, os.path.expanduser(v))
107
108    def __init__(self, usage=None, version=None):
109        OptionParser.__init__(self)
110
111
112def read_simple_accessdb(fn, auth, mask=[]):
113    """
114    Read a simple access database.  Each line is a fedid (of the form
115    fedid:hexstring) and a comma separated list of atributes to be assigned to
116    it.  This parses out the fedids and adds the attributes to the authorizer.
117    comments (preceded with a #) and blank lines are ignored.  Exceptions (e.g.
118    file exceptions and ValueErrors from badly parsed lines) are propagated.
119    """
120
121    rv = [ ]
122    lineno = 0
123    fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\
124            "(\w+\s*(,\s*\w+)*)")
125
126    # If a single string came in, make it a list
127    if isinstance(mask, basestring): mask = [ mask ]
128
129    f = open(fn, 'r')
130    for line in f:
131        lineno += 1
132        line = line.strip()
133        if line.startswith('#') or len(line) == 0: 
134            continue
135        m = fedid_line.match(line)
136        if m :
137            fid = fedid(hexstr=m.group(1))
138            for a in [ a.strip() for a in m.group(2).split(",") \
139                    if not mask or a.strip() in mask ]:
140                auth.set_attribute(fid, a.strip())
141        else:
142            raise ValueError("Badly formatted line in accessdb: %s line %d" %\
143                    (fn, lineno))
144    f.close()
145    return rv
146       
147
148def pack_id(id):
149    """
150    Return a dictionary with the field name set by the id type.  Handy for
151    creating dictionaries to be converted to messages.
152    """
153    if isinstance(id, fedid): return { 'fedid': id }
154    elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id }
155    else: return { 'localname': id}
156
157def unpack_id(id):
158    """return id as a type determined by the key"""
159    for k in ("localname", "fedid", "uri", "kerberosUsername"):
160        if id.has_key(k): return id[k]
161    return None
162
163def set_log_level(config, sect, log):
164    """ Set the logging level to the value passed in sect of config."""
165    # The getattr sleight of hand finds the logging level constant
166    # corrersponding to the string.  We're a little paranoid to avoid user
167    # mayhem.
168    if config.has_option(sect, "log_level"):
169        level_str = config.get(sect, "log_level")
170        try:
171            level = int(getattr(logging, level_str.upper(), -1))
172
173            if  logging.DEBUG <= level <= logging.CRITICAL:
174                log.setLevel(level)
175            else:
176                log.error("Bad experiment_log value: %s" % level_str)
177
178        except ValueError:
179            log.error("Bad experiment_log value: %s" % level_str)
180
181def copy_file(src, dest, size=1024):
182    """
183    Exceedingly simple file copy.  Throws an EnvironmentError if there's a problem.
184    """
185    s = open(src,'r')
186    d = open(dest, 'w')
187
188    buf = s.read(size)
189    while buf != "":
190        d.write(buf)
191        buf = s.read(size)
192    s.close()
193    d.close()
194
195def get_url(url, cf, destdir, fn=None, max_retries=5):
196    """
197    Get data from a federated data store.  This presents the client cert/fedid
198    to the http server.  We retry up to max_retries times.
199    """
200    po = urlparse(url)
201    if not fn:
202        fn = po.path.rpartition('/')[2]
203    retries = 0
204    ok = False
205    failed_exception = None
206    while not ok and retries < 5:
207        try:
208            conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
209                    cert_file=cf, key_file=cf)
210            conn.putrequest('GET', po.path)
211            conn.endheaders()
212            response = conn.getresponse()
213
214            lf = open("%s/%s" % (destdir, fn), "w")
215            buf = response.read(4096)
216            while buf:
217                lf.write(buf)
218                buf = response.read(4096)
219            lf.close()
220            ok = True
221        except EnvironmentError, e:
222            failed_excpetion = e
223            retries += 1
224        except httplib.HTTPException, e:
225            failed_exception = e
226            retries += 1
227        except sslerror, e:
228            failed_exception = e
229            retries += 1
230        except SSLError, e:
231            failed_exception = e
232            retries += 1
233
234    if retries > 5 and failed_exception:
235        raise failed_excpetion
236
237# Functions to manipulate composite testbed names
238def testbed_base(tb):
239    """
240    Simple code to get the base testebd name.
241    """
242    i = tb.find('/')
243    if i == -1: return tb
244    else: return tb[0:i]
245
246def testbed_suffix(tb):
247    """
248    Simple code to get a testbed suffix, if nay.  No suffix returns None.
249    """
250    i = tb.find('/')
251    if i != -1: return tb[i+1:]
252    else: return None
253
254def split_testbed(tb):
255    """
256    Return a testbed and a suffix as a tuple.  No suffix returns None for that
257    field
258    """
259
260    i = tb.find('/')
261    if i != -1: return (tb[0:i], tb[i+1:])
262    else: return (tb, None)
263
264def join_testbed(base, suffix=None):
265    """
266    Build a testbed with suffix.  If base is itself a tuple, combine them,
267    otherwise combine the two.
268    """
269    if isinstance(base, tuple):
270        if len(base) == 2:
271            return '/'.join(base)
272        else:
273            raise RuntimeError("Too many tuple elements for join_testbed")
274    else:
275        if suffix:
276            return '/'.join((base, suffix))
277        else:
278            return base
279
280def abac_pem_type(cert):
281    key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$')
282    cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') 
283    type = None
284
285    f = open(cert, 'r')
286    for line in f:
287        if key_re.match(line):
288            if type is None: type = 'key'
289            elif type == 'cert': type = 'both'
290        elif cert_re.match(line):
291            if type is None: type = 'cert'
292            elif type == 'key': type = 'both'
293        if type == 'both': break
294    f.close()
295    return type
296
297def abac_split_cert(cert, keyfile=None, certfile=None):
298    """
299    Split the certificate file in cert into a certificate file and a key file
300    in cf and kf respectively.  The ABAC tools generally cannot handle combined
301    certificates/keys.  If kf anc cf are given, they are used, otherwise tmp
302    files are created.  Created tmp files must be deleted.  Problems opening or
303    writing files will cause exceptions.
304    """
305    class diversion:
306        '''
307        Wraps up the reqular expression to start and end a diversion, as well as
308        the open file that gets the lines.  If fd is passed in, use that system
309        file (probably from a mkstemp.  Otherwise open the given filename.
310        '''
311        def __init__(self, start, end, fn=None, fd=None):
312            self.start = re.compile(start)
313            self.end = re.compile(end)
314
315            if not fd:
316                # Open the file securely with minimal permissions. NB file
317                # cannot exist before this call.
318                fd = os.open(fn,
319                    (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600)
320
321            self.f = os.fdopen(fd, 'w')
322
323        def close(self):
324            self.f.close()
325
326    if not keyfile:
327        kf, rkeyfile = mkstemp(suffix=".pem")
328    else:
329        kf, rkeyfile = None, keyfile
330
331    if not certfile:
332        cf, rcertfile = mkstemp(suffix=".pem")
333    else:
334        cf, rcertfile = None, certfile
335
336    # Initialize the diversions
337    divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in (
338        ('\s*-----BEGIN RSA PRIVATE KEY-----$', 
339            '\s*-----END RSA PRIVATE KEY-----$',
340            keyfile, kf),
341        ('\s*-----BEGIN CERTIFICATE-----$', 
342            '\s*-----END CERTIFICATE-----$',
343            certfile, cf))]
344
345    # walk through the file, beginning a diversion when a start regexp
346    # matches until the end regexp matches.  While in the two regexps,
347    # print each line to the open diversion file (including the two
348    # matches).
349    active = None
350    f = open(cert, 'r')
351    for l in f:
352        if active:
353            if active.end.match(l):
354                print >>active.f, l,
355                active = None
356        else:
357            for d in divs:
358                if d.start.match(l):
359                    active = d
360                    break
361        if active: print >>active.f, l,
362
363    # This is probably unnecessary.  Close all the diversion files.
364    for d in divs: d.close()
365    return rkeyfile, rcertfile
366
367def abac_context_to_creds(context):
368    """
369    Pull all the credentials out of the context and return 2  lists of the
370    underlying credentials in an exportable format, IDs and attributes.
371    There are no duplicates in the lists.
372    """
373    ids, attrs = set(), set()
374    # This should be a one-iteration loop
375    for c in context.credentials():
376        ids.add(str(c.issuer_cert()))
377        attrs.add(str(c.attribute_cert()))
378
379    return list(ids), list(attrs)
380
381def find_pickle_problem(o, st=None):
382    """
383    Debugging routine to figure out what doesn't pickle from a dict full of
384    dicts and lists.  It tries to walk down the lists and dicts and pickle each
385    atom.  If something fails to pickle, it prints an approximation of a stack
386    trace through the data structure.
387    """
388    if st is None: st = [ ]
389
390    if isinstance(o, dict):
391        for k, i in o.items():
392            st.append(k)
393            find_pickle_problem(i, st)
394            st.pop()
395    elif isinstance(o, list):
396        st.append('list')
397        for i in o:
398            find_pickle_problem(i, st)
399        st.pop()
400    else:
401        try:
402            pickle.dumps(o)
403        except pickle.PicklingError, e:
404            print >>sys.stderr, "<PicklingError>"
405            print >>sys.stderr, st
406            print >>sys.stderr, o
407            print >>sys.stderr, e
408            print >>sys.stderr, "</PicklingError>"
409
Note: See TracBrowser for help on using the repository browser.