source: fedd/federation/util.py @ c573278

axis_examplecompt_changesinfo-ops
Last change on this file since c573278 was c573278, checked in by Ted Faber <faber@…>, 13 years ago

Checkpoint. Still lots to do

  • Property mode set to 100644
File size: 11.5 KB
Line 
1#!/usr/local/bin/python
2
3import re
4import os
5import string
6import logging
7import pickle
8
9import httplib
10
11from socket import sslerror
12from tempfile import mkstemp
13
14from M2Crypto import SSL
15from M2Crypto.SSL import SSLError
16from fedid import fedid
17from service_error import service_error
18from urlparse import urlparse
19
20
21# If this is an old enough version of M2Crypto.SSL that has an
22# ssl_verify_callback that doesn't allow 0-length signed certs, create a
23# version of that callback that does.  This is edited from the original in
24# M2Crypto.SSL.cb.  This version also elides the printing to stderr.
25if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None):
26    from M2Crypto.SSL.Context import map
27    from M2Crypto import m2
28
29    def ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
30        unknown_issuer = [
31            m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY,
32            m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE,
33            m2.X509_V_ERR_CERT_UNTRUSTED,
34            m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
35            ]
36        ssl_ctx = map()[ssl_ctx_ptr]
37
38        if errnum in unknown_issuer: 
39            if ssl_ctx.get_allow_unknown_ca():
40                ok = 1
41        # CRL checking goes here...
42        if ok:
43            if ssl_ctx.get_verify_depth() >= errdepth:
44                ok = 1
45            else:
46                ok = 0
47        return ok
48else:
49    def ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
50        raise ValueError("This should never be called")
51
52class fedd_ssl_context(SSL.Context):
53    """
54    Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd.
55    """
56    def __init__(self, my_cert, trusted_certs=None, password=None):
57        """
58        construct a fedd_ssl_context
59
60        @param my_cert: PEM file with my certificate in it
61        @param trusted_certs: PEM file with trusted certs in it (optional)
62        """
63        SSL.Context.__init__(self)
64
65        # load_cert takes a callback to get a password, not a password, so if
66        # the caller provided a password, this creates a nonce callback using a
67        # lambda form.
68        if password != None and not callable(password):
69            # This is cute.  password = lambda *args: password produces a
70            # function object that returns itself rather than one that returns
71            # the object itself.  This is because password is an object
72            # reference and after the assignment it's a lambda.  So we assign
73            # to a temp.
74            pwd = str(password)
75            password =lambda *args: pwd
76
77        # The calls to str below (and above) are because the underlying SSL
78        # stuff is intolerant of unicode.
79        if password != None:
80            self.load_cert(str(my_cert), callback=password)
81        else:
82            self.load_cert(str(my_cert))
83
84        # If no trusted certificates are specified, allow unknown CAs.
85        if trusted_certs: 
86            self.load_verify_locations(trusted_certs)
87            self.set_verify(SSL.verify_peer, 10)
88        else:
89            # More legacy code.  Recent versions of M2Crypto express the
90            # allow_unknown_ca option through a callback turned to allow it.
91            # Older versions use a standard callback that respects the
92            # attribute.  This should work under both regines.
93            callb = getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', 
94                    ssl_verify_callback)
95            self.set_allow_unknown_ca(True)
96            self.set_verify(SSL.verify_peer, 10, callback=callb)
97
98def read_simple_accessdb(fn, auth, mask=[]):
99    """
100    Read a simple access database.  Each line is a fedid (of the form
101    fedid:hexstring) and a comma separated list of atributes to be assigned to
102    it.  This parses out the fedids and adds the attributes to the authorizer.
103    comments (preceded with a #) and blank lines are ignored.  Exceptions (e.g.
104    file exceptions and ValueErrors from badly parsed lines) are propagated.
105    """
106
107    rv = [ ]
108    lineno = 0
109    fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\
110            "(\w+\s*(,\s*\w+)*)")
111
112    # If a single string came in, make it a list
113    if isinstance(mask, basestring): mask = [ mask ]
114
115    f = open(fn, 'r')
116    for line in f:
117        lineno += 1
118        line = line.strip()
119        if line.startswith('#') or len(line) == 0: 
120            continue
121        m = fedid_line.match(line)
122        if m :
123            fid = fedid(hexstr=m.group(1))
124            for a in [ a.strip() for a in m.group(2).split(",") \
125                    if not mask or a.strip() in mask ]:
126                auth.set_attribute(fid, a.strip())
127        else:
128            raise ValueError("Badly formatted line in accessdb: %s line %d" %\
129                    (fn, lineno))
130    f.close()
131    return rv
132       
133
134def pack_id(id):
135    """
136    Return a dictionary with the field name set by the id type.  Handy for
137    creating dictionaries to be converted to messages.
138    """
139    if isinstance(id, fedid): return { 'fedid': id }
140    elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id }
141    else: return { 'localname': id}
142
143def unpack_id(id):
144    """return id as a type determined by the key"""
145    for k in ("localname", "fedid", "uri", "kerberosUsername"):
146        if id.has_key(k): return id[k]
147    return None
148
149def set_log_level(config, sect, log):
150    """ Set the logging level to the value passed in sect of config."""
151    # The getattr sleight of hand finds the logging level constant
152    # corrersponding to the string.  We're a little paranoid to avoid user
153    # mayhem.
154    if config.has_option(sect, "log_level"):
155        level_str = config.get(sect, "log_level")
156        try:
157            level = int(getattr(logging, level_str.upper(), -1))
158
159            if  logging.DEBUG <= level <= logging.CRITICAL:
160                log.setLevel(level)
161            else:
162                log.error("Bad experiment_log value: %s" % level_str)
163
164        except ValueError:
165            log.error("Bad experiment_log value: %s" % level_str)
166
167def copy_file(src, dest, size=1024):
168    """
169    Exceedingly simple file copy.  Throws an EnvironmentError if there's a problem.
170    """
171    s = open(src,'r')
172    d = open(dest, 'w')
173
174    buf = s.read(size)
175    while buf != "":
176        d.write(buf)
177        buf = s.read(size)
178    s.close()
179    d.close()
180
181def get_url(url, cf, destdir, fn=None, max_retries=5):
182    """
183    Get data from a federated data store.  This presents the client cert/fedid
184    to the http server.  We retry up to max_retries times.
185    """
186    po = urlparse(url)
187    if not fn:
188        fn = po.path.rpartition('/')[2]
189    retries = 0
190    ok = False
191    failed_exception = None
192    while not ok and retries < 5:
193        try:
194            conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
195                    cert_file=cf, key_file=cf)
196            conn.putrequest('GET', po.path)
197            conn.endheaders()
198            response = conn.getresponse()
199
200            lf = open("%s/%s" % (destdir, fn), "w")
201            buf = response.read(4096)
202            while buf:
203                lf.write(buf)
204                buf = response.read(4096)
205            lf.close()
206            ok = True
207        except EnvironmentError, e:
208            failed_excpetion = e
209            retries += 1
210        except httplib.HTTPException, e:
211            failed_exception = e
212            retries += 1
213        except sslerror, e:
214            failed_exception = e
215            retries += 1
216        except SSLError, e:
217            failed_exception = e
218            retries += 1
219
220    if retries > 5 and failed_exception:
221        raise failed_excpetion
222
223# Functions to manipulate composite testbed names
224def testbed_base(tb):
225    """
226    Simple code to get the base testebd name.
227    """
228    i = tb.find('/')
229    if i == -1: return tb
230    else: return tb[0:i]
231
232def testbed_suffix(tb):
233    """
234    Simple code to get a testbed suffix, if nay.  No suffix returns None.
235    """
236    i = tb.find('/')
237    if i != -1: return tb[i+1:]
238    else: return None
239
240def split_testbed(tb):
241    """
242    Return a testbed and a suffix as a tuple.  No suffix returns None for that
243    field
244    """
245
246    i = tb.find('/')
247    if i != -1: return (tb[0:i], tb[i+1:])
248    else: return (tb, None)
249
250def join_testbed(base, suffix=None):
251    """
252    Build a testbed with suffix.  If base is itself a tuple, combine them,
253    otherwise combine the two.
254    """
255    if isinstance(base, tuple):
256        if len(base) == 2:
257            return '/'.join(base)
258        else:
259            raise RuntimeError("Too many tuple elements for join_testbed")
260    else:
261        if suffix:
262            return '/'.join((base, suffix))
263        else:
264            return base
265
266def abac_pem_type(cert):
267    key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$')
268    cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') 
269    type = None
270
271    f = open(cert, 'r')
272    for line in f:
273        if key_re.match(line):
274            if type is None: type = 'key'
275            elif type == 'cert': type = 'both'
276        elif cert_re.match(line):
277            if type is None: type = 'cert'
278            elif type == 'key': type = 'both'
279        if type == 'both': break
280    f.close()
281    return type
282
283def abac_split_cert(cert, keyfile=None, certfile=None):
284    """
285    Split the certificate file in cert into a certificate file and a key file
286    in cf and kf respectively.  The ABAC tools generally cannot handle combined
287    certificates/keys.  If kf anc cf are given, they are used, otherwise tmp
288    files are created.  Created tmp files must be deleted.  Problems opening or
289    writing files will cause exceptions.
290    """
291    class diversion:
292        '''
293        Wraps up the reqular expression to start and end a diversion, as well as
294        the open file that gets the lines.  If fd is passed in, use that system
295        file (probably from a mkstemp.  Otherwise open the given filename.
296        '''
297        def __init__(self, start, end, fn=None, fd=None):
298            self.start = re.compile(start)
299            self.end = re.compile(end)
300
301            if not fd:
302                # Open the file securely with minimal permissions. NB file
303                # cannot exist before this call.
304                fd = os.open(fn,
305                    (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600)
306
307            self.f = os.fdopen(fd, 'w')
308
309        def close(self):
310            self.f.close()
311
312    if not keyfile:
313        kf, rkeyfile = mkstemp(suffix=".pem")
314    else:
315        kf, rkeyfile = None, keyfile
316
317    if not certfile:
318        cf, rcertfile = mkstemp(suffix=".pem")
319    else:
320        cf, rcertfile = None, certfile
321
322    # Initialize the diversions
323    divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in (
324        ('\s*-----BEGIN RSA PRIVATE KEY-----$', 
325            '\s*-----END RSA PRIVATE KEY-----$',
326            keyfile, kf),
327        ('\s*-----BEGIN CERTIFICATE-----$', 
328            '\s*-----END CERTIFICATE-----$',
329            certfile, cf))]
330
331    # walk through the file, beginning a diversion when a start regexp
332    # matches until the end regexp matches.  While in the two regexps,
333    # print each line to the open diversion file (including the two
334    # matches).
335    active = None
336    f = open(cert, 'r')
337    for l in f:
338        if active:
339            if active.end.match(l):
340                print >>active.f, l,
341                active = None
342        else:
343            for d in divs:
344                if d.start.match(l):
345                    active = d
346                    break
347        if active: print >>active.f, l,
348
349    # This is probably unnecessary.  Close all the diversion files.
350    for d in divs: d.close()
351    return rkeyfile, rcertfile
352
353def abac_context_to_creds(context):
354    """
355    Pull all the credentials out of the context and return 2  lists of the
356    underlying credentials in an exportable format, IDs and attributes.
357    There are no duplicates in the lists.
358    """
359    ids, attrs = set(), set()
360    # This should be a one-iteration loop
361    for c in context.credentials():
362        ids.add(c.issuer_cert())
363        attrs.add(c.attribute_cert())
364
365    return list(ids), list(attrs)
366
367def find_pickle_problem(o, st=None):
368    """
369    Debugging routine to figure out what doesn't pickle from a dict full of
370    dicts and lists.  It tries to walk down the lists and dicts and pickle each
371    atom.  If something fails to pickle, it prints an approximation of a stack
372    trace through the data structure.
373    """
374    if st is None: st = [ ]
375
376    if isinstance(o, dict):
377        for k, i in o.items():
378            st.append(k)
379            find_pickle_problem(i, st)
380            st.pop()
381    elif isinstance(o, list):
382        st.append('list')
383        for i in o:
384            find_pickle_problem(i, st)
385        st.pop()
386    else:
387        try:
388            pickle.dumps(o)
389        except pickle.PicklingError, e:
390            print >>sys.stderr, "<PicklingError>"
391            print >>sys.stderr, st
392            print >>sys.stderr, o
393            print >>sys.stderr, e
394            print >>sys.stderr, "</PicklingError>"
395
Note: See TracBrowser for help on using the repository browser.