source: fedd/federation/util.py @ e11f4e0

axis_examplecompt_changesinfo-opsversion-3.01version-3.02
Last change on this file since e11f4e0 was d3c8759, checked in by Ted Faber <faber@…>, 15 years ago

Wholesale change of IOError to EnvironmentError? for file operations. Lots of
uncaught EnvironmentErrors? were causing spurious error conditions, e.g. on disk
full.

  • Property mode set to 100644
File size: 8.5 KB
Line 
1#!/usr/local/bin/python
2
3import re
4import string
5import logging
6import pickle
7
8import httplib
9
10from socket import sslerror
11
12from M2Crypto import SSL
13from M2Crypto.SSL import SSLError
14from fedid import fedid
15from service_error import service_error
16from urlparse import urlparse
17
18
19# If this is an old enough version of M2Crypto.SSL that has an
20# ssl_verify_callback that doesn't allow 0-length signed certs, create a
21# version of that callback that does.  This is edited from the original in
22# M2Crypto.SSL.cb.  This version also elides the printing to stderr.
23if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None):
24    from M2Crypto.SSL.Context import map
25    from M2Crypto import m2
26
27    def ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
28        unknown_issuer = [
29            m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY,
30            m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE,
31            m2.X509_V_ERR_CERT_UNTRUSTED,
32            m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT
33            ]
34        ssl_ctx = map()[ssl_ctx_ptr]
35
36        if errnum in unknown_issuer: 
37            if ssl_ctx.get_allow_unknown_ca():
38                ok = 1
39        # CRL checking goes here...
40        if ok:
41            if ssl_ctx.get_verify_depth() >= errdepth:
42                ok = 1
43            else:
44                ok = 0
45        return ok
46else:
47    def ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok):
48        raise ValueError("This should never be called")
49
50class fedd_ssl_context(SSL.Context):
51    """
52    Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd.
53    """
54    def __init__(self, my_cert, trusted_certs=None, password=None):
55        """
56        construct a fedd_ssl_context
57
58        @param my_cert: PEM file with my certificate in it
59        @param trusted_certs: PEM file with trusted certs in it (optional)
60        """
61        SSL.Context.__init__(self)
62
63        # load_cert takes a callback to get a password, not a password, so if
64        # the caller provided a password, this creates a nonce callback using a
65        # lambda form.
66        if password != None and not callable(password):
67            # This is cute.  password = lambda *args: password produces a
68            # function object that returns itself rather than one that returns
69            # the object itself.  This is because password is an object
70            # reference and after the assignment it's a lambda.  So we assign
71            # to a temp.
72            pwd = str(password)
73            password =lambda *args: pwd
74
75        # The calls to str below (and above) are because the underlying SSL
76        # stuff is intolerant of unicode.
77        if password != None:
78            self.load_cert(str(my_cert), callback=password)
79        else:
80            self.load_cert(str(my_cert))
81
82        # If no trusted certificates are specified, allow unknown CAs.
83        if trusted_certs: 
84            self.load_verify_locations(trusted_certs)
85            self.set_verify(SSL.verify_peer, 10)
86        else:
87            # More legacy code.  Recent versions of M2Crypto express the
88            # allow_unknown_ca option through a callback turned to allow it.
89            # Older versions use a standard callback that respects the
90            # attribute.  This should work under both regines.
91            callb = getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', 
92                    ssl_verify_callback)
93            self.set_allow_unknown_ca(True)
94            self.set_verify(SSL.verify_peer, 10, callback=callb)
95
96def read_simple_accessdb(fn, auth, mask=[]):
97    """
98    Read a simple access database.  Each line is a fedid (of the form
99    fedid:hexstring) and a comma separated list of atributes to be assigned to
100    it.  This parses out the fedids and adds the attributes to the authorizer.
101    comments (preceded with a #) and blank lines are ignored.  Exceptions (e.g.
102    file exceptions and ValueErrors from badly parsed lines) are propagated.
103    """
104
105    rv = [ ]
106    lineno = 0
107    fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\
108            "(\w+\s*(,\s*\w+)*)")
109
110    # If a single string came in, make it a list
111    if isinstance(mask, basestring): mask = [ mask ]
112
113    f = open(fn, 'r')
114    for line in f:
115        lineno += 1
116        line = line.strip()
117        if line.startswith('#') or len(line) == 0: 
118            continue
119        m = fedid_line.match(line)
120        if m :
121            fid = fedid(hexstr=m.group(1))
122            for a in [ a.strip() for a in m.group(2).split(",") \
123                    if not mask or a.strip() in mask ]:
124                auth.set_attribute(fid, a.strip())
125        else:
126            raise ValueError("Badly formatted line in accessdb: %s line %d" %\
127                    (fn, lineno))
128    f.close()
129    return rv
130       
131
132def pack_id(id):
133    """
134    Return a dictionary with the field name set by the id type.  Handy for
135    creating dictionaries to be converted to messages.
136    """
137    if isinstance(id, fedid): return { 'fedid': id }
138    elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id }
139    else: return { 'localname': id}
140
141def unpack_id(id):
142    """return id as a type determined by the key"""
143    for k in ("localname", "fedid", "uri", "kerberosUsername"):
144        if id.has_key(k): return id[k]
145    return None
146
147def set_log_level(config, sect, log):
148    """ Set the logging level to the value passed in sect of config."""
149    # The getattr sleight of hand finds the logging level constant
150    # corrersponding to the string.  We're a little paranoid to avoid user
151    # mayhem.
152    if config.has_option(sect, "log_level"):
153        level_str = config.get(sect, "log_level")
154        try:
155            level = int(getattr(logging, level_str.upper(), -1))
156
157            if  logging.DEBUG <= level <= logging.CRITICAL:
158                log.setLevel(level)
159            else:
160                log.error("Bad experiment_log value: %s" % level_str)
161
162        except ValueError:
163            log.error("Bad experiment_log value: %s" % level_str)
164
165def copy_file(src, dest, size=1024):
166    """
167    Exceedingly simple file copy.  Throws an EnvironmentError if there's a problem.
168    """
169    s = open(src,'r')
170    d = open(dest, 'w')
171
172    buf = s.read(size)
173    while buf != "":
174        d.write(buf)
175        buf = s.read(size)
176    s.close()
177    d.close()
178
179def get_url(url, cf, destdir, fn=None, max_retries=5):
180    """
181    Get data from a federated data store.  This presents the client cert/fedid
182    to the http server.  We retry up to max_retries times.
183    """
184    po = urlparse(url)
185    if not fn:
186        fn = po.path.rpartition('/')[2]
187    retries = 0
188    ok = False
189    failed_exception = None
190    while not ok and retries < 5:
191        try:
192            conn = httplib.HTTPSConnection(po.hostname, port=po.port, 
193                    cert_file=cf, key_file=cf)
194            conn.putrequest('GET', po.path)
195            conn.endheaders()
196            response = conn.getresponse()
197
198            lf = open("%s/%s" % (destdir, fn), "w")
199            buf = response.read(4096)
200            while buf:
201                lf.write(buf)
202                buf = response.read(4096)
203            lf.close()
204            ok = True
205        except EnvironmentError, e:
206            failed_excpetion = e
207            retries += 1
208        except httplib.HTTPException, e:
209            failed_exception = e
210            retries += 1
211        except sslerror, e:
212            failed_exception = e
213            retries += 1
214        except SSLError, e:
215            failed_exception = e
216            retries += 1
217
218    if retries > 5 and failed_exception:
219        raise failed_excpetion
220
221# Functions to manipulate composite testbed names
222def testbed_base(tb):
223    """
224    Simple code to get the base testebd name.
225    """
226    i = tb.find('/')
227    if i == -1: return tb
228    else: return tb[0:i]
229
230def testbed_suffix(tb):
231    """
232    Simple code to get a testbed suffix, if nay.  No suffix returns None.
233    """
234    i = tb.find('/')
235    if i != -1: return tb[i+1:]
236    else: return None
237
238def split_testbed(tb):
239    """
240    Return a testbed and a suffix as a tuple.  No suffix returns None for that
241    field
242    """
243
244    i = tb.find('/')
245    if i != -1: return (tb[0:i], tb[i+1:])
246    else: return (tb, None)
247
248def join_testbed(base, suffix=None):
249    """
250    Build a testbed with suffix.  If base is itself a tuple, combine them,
251    otherwise combine the two.
252    """
253    if isinstance(base, tuple):
254        if len(base) == 2:
255            return '/'.join(base)
256        else:
257            raise RuntimeError("Too many tuple elements for join_testbed")
258    else:
259        if suffix:
260            return '/'.join((base, suffix))
261        else:
262            return base
263
264def find_pickle_problem(o, st=None):
265    """
266    Debugging routine to figure out what doesn't pickle from a dict full of
267    dicts and lists.  It tries to walk down the lists and dicts and pickle each
268    atom.  If something fails to pickle, it prints an approximation of a stack
269    trace through the data structure.
270    """
271    if st is None: st = [ ]
272
273    if isinstance(o, dict):
274        for k, i in o.items():
275            st.append(k)
276            find_pickle_problem(i, st)
277            st.pop()
278    elif isinstance(o, list):
279        st.append('list')
280        for i in o:
281            find_pickle_problem(i, st)
282        st.pop()
283    else:
284        try:
285            pickle.dumps(o)
286        except pickle.PicklingError, e:
287            print >>sys.stderr, "<PicklingError>"
288            print >>sys.stderr, st
289            print >>sys.stderr, o
290            print >>sys.stderr, e
291            print >>sys.stderr, "</PicklingError>"
292
Note: See TracBrowser for help on using the repository browser.