1 | #!/usr/local/bin/python |
---|
2 | |
---|
3 | import re |
---|
4 | import os |
---|
5 | import string |
---|
6 | import logging |
---|
7 | import pickle |
---|
8 | |
---|
9 | import httplib |
---|
10 | import ssl |
---|
11 | |
---|
12 | from optparse import OptionParser |
---|
13 | |
---|
14 | from socket import sslerror |
---|
15 | from tempfile import mkstemp |
---|
16 | |
---|
17 | from M2Crypto import SSL |
---|
18 | from M2Crypto.SSL import SSLError |
---|
19 | from deter import fedid |
---|
20 | from service_error import service_error |
---|
21 | from urlparse import urlparse |
---|
22 | from M2Crypto import m2 |
---|
23 | |
---|
24 | |
---|
25 | # If this is an old enough version of M2Crypto.SSL that has an |
---|
26 | # ssl_verify_callback that doesn't allow 0-length signed certs, create a |
---|
27 | # version of that callback that does. This is edited from the original in |
---|
28 | # M2Crypto.SSL.cb. This version also elides the printing to stderr. |
---|
29 | if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None): |
---|
30 | from M2Crypto.SSL.Context import map |
---|
31 | |
---|
32 | def fedd_ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok): |
---|
33 | unknown_issuer = [ |
---|
34 | m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY, |
---|
35 | m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE, |
---|
36 | m2.X509_V_ERR_CERT_UNTRUSTED, |
---|
37 | m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT |
---|
38 | ] |
---|
39 | # m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed |
---|
40 | if getattr(m2, 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None): |
---|
41 | unknown_issuer.append(getattr(m2, |
---|
42 | 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None)) |
---|
43 | ssl_ctx = map()[ssl_ctx_ptr] |
---|
44 | |
---|
45 | if errnum in unknown_issuer: |
---|
46 | if ssl_ctx.get_allow_unknown_ca(): |
---|
47 | ok = 1 |
---|
48 | # CRL checking goes here... |
---|
49 | if ok: |
---|
50 | if ssl_ctx.get_verify_depth() >= errdepth: |
---|
51 | ok = 1 |
---|
52 | else: |
---|
53 | ok = 0 |
---|
54 | return ok |
---|
55 | else: |
---|
56 | def fedd_ssl_verify_callback(ok, store): |
---|
57 | ''' |
---|
58 | m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed |
---|
59 | ''' |
---|
60 | errnum = store.get_error() |
---|
61 | if errnum == m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN: |
---|
62 | ok = 1 |
---|
63 | return ok |
---|
64 | else: |
---|
65 | return SSL.cb.ssl_verify_callback_allow_unknown_ca(ok, store) |
---|
66 | |
---|
67 | class fedd_ssl_context(SSL.Context): |
---|
68 | """ |
---|
69 | Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd. |
---|
70 | """ |
---|
71 | def __init__(self, my_cert, trusted_certs=None, password=None, protocol=None): |
---|
72 | """ |
---|
73 | construct a fedd_ssl_context |
---|
74 | |
---|
75 | @param my_cert: PEM file with my certificate in it |
---|
76 | @param trusted_certs: PEM file with trusted certs in it (optional) |
---|
77 | """ |
---|
78 | if protocol: SSL.Context.__init__(self, protocol=protocol) |
---|
79 | else: SSL.Context.__init__(self) |
---|
80 | |
---|
81 | # load_cert takes a callback to get a password, not a password, so if |
---|
82 | # the caller provided a password, this creates a nonce callback using a |
---|
83 | # lambda form. |
---|
84 | if password != None and not callable(password): |
---|
85 | # This is cute. password = lambda *args: password produces a |
---|
86 | # function object that returns itself rather than one that returns |
---|
87 | # the object itself. This is because password is an object |
---|
88 | # reference and after the assignment it's a lambda. So we assign |
---|
89 | # to a temp. |
---|
90 | pwd = str(password) |
---|
91 | password =lambda *args: pwd |
---|
92 | |
---|
93 | # The calls to str below (and above) are because the underlying SSL |
---|
94 | # stuff is intolerant of unicode. |
---|
95 | if password != None: |
---|
96 | self.load_cert(str(my_cert), callback=password) |
---|
97 | else: |
---|
98 | self.load_cert(str(my_cert)) |
---|
99 | |
---|
100 | # If no trusted certificates are specified, allow unknown CAs. |
---|
101 | if trusted_certs: |
---|
102 | self.load_verify_locations(trusted_certs) |
---|
103 | self.set_verify(SSL.verify_peer, 10) |
---|
104 | else: |
---|
105 | # Install the proper callback to allow self-signed certs |
---|
106 | self.set_allow_unknown_ca(True) |
---|
107 | self.set_verify(SSL.verify_peer, 10, |
---|
108 | callback=fedd_ssl_verify_callback) |
---|
109 | |
---|
110 | # no session caching |
---|
111 | self.set_session_cache_mode(0) |
---|
112 | |
---|
113 | class file_expanding_opts(OptionParser): |
---|
114 | def expand_file(self, option, opt_str, v, p): |
---|
115 | """ |
---|
116 | Store the given value to the given destination after expanding home |
---|
117 | directories. |
---|
118 | """ |
---|
119 | setattr(p.values, option.dest, os.path.expanduser(v)) |
---|
120 | |
---|
121 | def __init__(self, usage=None, version=None): |
---|
122 | OptionParser.__init__(self) |
---|
123 | |
---|
124 | |
---|
125 | def read_simple_accessdb(fn, auth, mask=[]): |
---|
126 | """ |
---|
127 | Read a simple access database. Each line is a fedid (of the form |
---|
128 | fedid:hexstring) and a comma separated list of atributes to be assigned to |
---|
129 | it. This parses out the fedids and adds the attributes to the authorizer. |
---|
130 | comments (preceded with a #) and blank lines are ignored. Exceptions (e.g. |
---|
131 | file exceptions and ValueErrors from badly parsed lines) are propagated. |
---|
132 | """ |
---|
133 | |
---|
134 | rv = [ ] |
---|
135 | lineno = 0 |
---|
136 | fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\ |
---|
137 | "(\w+\s*(,\s*\w+)*)") |
---|
138 | |
---|
139 | # If a single string came in, make it a list |
---|
140 | if isinstance(mask, basestring): mask = [ mask ] |
---|
141 | |
---|
142 | f = open(fn, 'r') |
---|
143 | for line in f: |
---|
144 | lineno += 1 |
---|
145 | line = line.strip() |
---|
146 | if line.startswith('#') or len(line) == 0: |
---|
147 | continue |
---|
148 | m = fedid_line.match(line) |
---|
149 | if m : |
---|
150 | fid = fedid(hexstr=m.group(1)) |
---|
151 | for a in [ a.strip() for a in m.group(2).split(",") \ |
---|
152 | if not mask or a.strip() in mask ]: |
---|
153 | auth.set_attribute(fid, a.strip()) |
---|
154 | else: |
---|
155 | raise ValueError("Badly formatted line in accessdb: %s line %d" %\ |
---|
156 | (fn, lineno)) |
---|
157 | f.close() |
---|
158 | return rv |
---|
159 | |
---|
160 | |
---|
161 | def pack_id(id): |
---|
162 | """ |
---|
163 | Return a dictionary with the field name set by the id type. Handy for |
---|
164 | creating dictionaries to be converted to messages. |
---|
165 | """ |
---|
166 | if isinstance(id, fedid): return { 'fedid': id } |
---|
167 | elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id } |
---|
168 | else: return { 'localname': id} |
---|
169 | |
---|
170 | def unpack_id(id): |
---|
171 | """return id as a type determined by the key""" |
---|
172 | for k in ("localname", "fedid", "uri", "kerberosUsername"): |
---|
173 | if id.has_key(k): return id[k] |
---|
174 | return None |
---|
175 | |
---|
176 | def set_log_level(config, sect, log): |
---|
177 | """ Set the logging level to the value passed in sect of config.""" |
---|
178 | # The getattr sleight of hand finds the logging level constant |
---|
179 | # corrersponding to the string. We're a little paranoid to avoid user |
---|
180 | # mayhem. |
---|
181 | if config.has_option(sect, "log_level"): |
---|
182 | level_str = config.get(sect, "log_level") |
---|
183 | try: |
---|
184 | level = int(getattr(logging, level_str.upper(), -1)) |
---|
185 | |
---|
186 | if logging.DEBUG <= level <= logging.CRITICAL: |
---|
187 | log.setLevel(level) |
---|
188 | else: |
---|
189 | log.error("Bad experiment_log value: %s" % level_str) |
---|
190 | |
---|
191 | except ValueError: |
---|
192 | log.error("Bad experiment_log value: %s" % level_str) |
---|
193 | |
---|
194 | def copy_file(src, dest, size=1024): |
---|
195 | """ |
---|
196 | Exceedingly simple file copy. Throws an EnvironmentError if there's a problem. |
---|
197 | """ |
---|
198 | s = open(src,'r') |
---|
199 | d = open(dest, 'w') |
---|
200 | |
---|
201 | buf = s.read(size) |
---|
202 | while buf != "": |
---|
203 | d.write(buf) |
---|
204 | buf = s.read(size) |
---|
205 | s.close() |
---|
206 | d.close() |
---|
207 | |
---|
208 | def get_url(url, cf, destdir, fn=None, max_retries=5, log=None): |
---|
209 | """ |
---|
210 | Get data from a federated data store. This presents the client cert/fedid |
---|
211 | to the http server. We retry up to max_retries times. |
---|
212 | """ |
---|
213 | po = urlparse(url) |
---|
214 | if not fn: |
---|
215 | fn = po.path.rpartition('/')[2] |
---|
216 | retries = 0 |
---|
217 | ok = False |
---|
218 | failed_exception = None |
---|
219 | # Build a context on modern systems, on old ones fall back to the old |
---|
220 | # HTTPSConnection definition. |
---|
221 | context = None |
---|
222 | try: |
---|
223 | # This context does not check the self-signed server cert |
---|
224 | context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) |
---|
225 | context.load_cert_chain(cf) |
---|
226 | except AttributeError: |
---|
227 | pass |
---|
228 | while not ok and retries < max_retries: |
---|
229 | try: |
---|
230 | if context is not None: |
---|
231 | conn = httplib.HTTPSConnection(po.hostname, port=po.port, |
---|
232 | timeout=30, context=context) |
---|
233 | else: |
---|
234 | conn = httplib.HTTPSConnection(po.hostname, port=po.port, |
---|
235 | cert_file=cf, key_file=cf, timeout=30) |
---|
236 | conn.putrequest('GET', po.path) |
---|
237 | conn.endheaders() |
---|
238 | response = conn.getresponse() |
---|
239 | |
---|
240 | lf = open("%s/%s" % (destdir, fn), "w") |
---|
241 | buf = response.read(4096) |
---|
242 | while buf: |
---|
243 | lf.write(buf) |
---|
244 | buf = response.read(4096) |
---|
245 | lf.close() |
---|
246 | ok = True |
---|
247 | except EnvironmentError, e: |
---|
248 | failed_exception = e |
---|
249 | retries += 1 |
---|
250 | except httplib.HTTPException, e: |
---|
251 | failed_exception = e |
---|
252 | retries += 1 |
---|
253 | except sslerror, e: |
---|
254 | failed_exception = e |
---|
255 | retries += 1 |
---|
256 | except SSLError, e: |
---|
257 | failed_exception = e |
---|
258 | retries += 1 |
---|
259 | |
---|
260 | if retries >= max_retries and failed_exception: |
---|
261 | if log: |
---|
262 | log.debug('Raising %s', failed_exception) |
---|
263 | raise failed_exception |
---|
264 | |
---|
265 | # Functions to manipulate composite testbed names |
---|
266 | def testbed_base(tb): |
---|
267 | """ |
---|
268 | Simple code to get the base testebd name. |
---|
269 | """ |
---|
270 | i = tb.find('/') |
---|
271 | if i == -1: return tb |
---|
272 | else: return tb[0:i] |
---|
273 | |
---|
274 | def testbed_suffix(tb): |
---|
275 | """ |
---|
276 | Simple code to get a testbed suffix, if any. No suffix returns None. |
---|
277 | """ |
---|
278 | i = tb.find('/') |
---|
279 | if i != -1: return tb[i+1:] |
---|
280 | else: return None |
---|
281 | |
---|
282 | def split_testbed(tb): |
---|
283 | """ |
---|
284 | Return a testbed and a suffix as a tuple. No suffix returns None for that |
---|
285 | field |
---|
286 | """ |
---|
287 | |
---|
288 | i = tb.find('/') |
---|
289 | if i != -1: return (tb[0:i], tb[i+1:]) |
---|
290 | else: return (tb, None) |
---|
291 | |
---|
292 | def join_testbed(base, suffix=None): |
---|
293 | """ |
---|
294 | Build a testbed with suffix. If base is itself a tuple, combine them, |
---|
295 | otherwise combine the two. |
---|
296 | """ |
---|
297 | if isinstance(base, tuple): |
---|
298 | if len(base) == 2: |
---|
299 | return '/'.join(base) |
---|
300 | else: |
---|
301 | raise RuntimeError("Too many tuple elements for join_testbed") |
---|
302 | else: |
---|
303 | if suffix: |
---|
304 | return '/'.join((base, suffix)) |
---|
305 | else: |
---|
306 | return base |
---|
307 | |
---|
308 | def abac_pem_type(cert): |
---|
309 | key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$') |
---|
310 | cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') |
---|
311 | type = None |
---|
312 | |
---|
313 | f = open(cert, 'r') |
---|
314 | for line in f: |
---|
315 | if key_re.match(line): |
---|
316 | if type is None: type = 'key' |
---|
317 | elif type == 'cert': type = 'both' |
---|
318 | elif cert_re.match(line): |
---|
319 | if type is None: type = 'cert' |
---|
320 | elif type == 'key': type = 'both' |
---|
321 | if type == 'both': break |
---|
322 | f.close() |
---|
323 | return type |
---|
324 | |
---|
325 | def abac_split_cert(cert, keyfile=None, certfile=None): |
---|
326 | """ |
---|
327 | Split the certificate file in cert into a certificate file and a key file |
---|
328 | in cf and kf respectively. The ABAC tools generally cannot handle combined |
---|
329 | certificates/keys. If kf anc cf are given, they are used, otherwise tmp |
---|
330 | files are created. Created tmp files must be deleted. Problems opening or |
---|
331 | writing files will cause exceptions. |
---|
332 | """ |
---|
333 | class diversion: |
---|
334 | ''' |
---|
335 | Wraps up the reqular expression to start and end a diversion, as well as |
---|
336 | the open file that gets the lines. If fd is passed in, use that system |
---|
337 | file (probably from a mkstemp. Otherwise open the given filename. |
---|
338 | ''' |
---|
339 | def __init__(self, start, end, fn=None, fd=None): |
---|
340 | self.start = re.compile(start) |
---|
341 | self.end = re.compile(end) |
---|
342 | |
---|
343 | if not fd: |
---|
344 | # Open the file securely with minimal permissions. NB file |
---|
345 | # cannot exist before this call. |
---|
346 | fd = os.open(fn, |
---|
347 | (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600) |
---|
348 | |
---|
349 | self.f = os.fdopen(fd, 'w') |
---|
350 | |
---|
351 | def close(self): |
---|
352 | self.f.close() |
---|
353 | |
---|
354 | if not keyfile: |
---|
355 | kf, rkeyfile = mkstemp(suffix=".pem") |
---|
356 | else: |
---|
357 | kf, rkeyfile = None, keyfile |
---|
358 | |
---|
359 | if not certfile: |
---|
360 | cf, rcertfile = mkstemp(suffix=".pem") |
---|
361 | else: |
---|
362 | cf, rcertfile = None, certfile |
---|
363 | |
---|
364 | # Initialize the diversions |
---|
365 | divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in ( |
---|
366 | ('\s*-----BEGIN RSA PRIVATE KEY-----$', |
---|
367 | '\s*-----END RSA PRIVATE KEY-----$', |
---|
368 | keyfile, kf), |
---|
369 | ('\s*-----BEGIN CERTIFICATE-----$', |
---|
370 | '\s*-----END CERTIFICATE-----$', |
---|
371 | certfile, cf))] |
---|
372 | |
---|
373 | # walk through the file, beginning a diversion when a start regexp |
---|
374 | # matches until the end regexp matches. While in the two regexps, |
---|
375 | # print each line to the open diversion file (including the two |
---|
376 | # matches). |
---|
377 | active = None |
---|
378 | f = open(cert, 'r') |
---|
379 | for l in f: |
---|
380 | if active: |
---|
381 | if active.end.match(l): |
---|
382 | print >>active.f, l, |
---|
383 | active = None |
---|
384 | else: |
---|
385 | for d in divs: |
---|
386 | if d.start.match(l): |
---|
387 | active = d |
---|
388 | break |
---|
389 | if active: print >>active.f, l, |
---|
390 | |
---|
391 | # This is probably unnecessary. Close all the diversion files. |
---|
392 | for d in divs: d.close() |
---|
393 | return rkeyfile, rcertfile |
---|
394 | |
---|
395 | def abac_context_to_creds(context): |
---|
396 | """ |
---|
397 | Pull all the credentials out of the context and return 2 lists of the |
---|
398 | underlying credentials in an exportable format, IDs and attributes. |
---|
399 | There are no duplicates in the lists. |
---|
400 | """ |
---|
401 | ids, attrs = set(), set() |
---|
402 | # This should be a one-iteration loop |
---|
403 | for c in context.credentials(): |
---|
404 | ids.add(str(c.issuer_cert())) |
---|
405 | attrs.add(str(c.attribute_cert())) |
---|
406 | |
---|
407 | return list(ids), list(attrs) |
---|
408 | |
---|
409 | def find_pickle_problem(o, st=None): |
---|
410 | """ |
---|
411 | Debugging routine to figure out what doesn't pickle from a dict full of |
---|
412 | dicts and lists. It tries to walk down the lists and dicts and pickle each |
---|
413 | atom. If something fails to pickle, it prints an approximation of a stack |
---|
414 | trace through the data structure. |
---|
415 | """ |
---|
416 | if st is None: st = [ ] |
---|
417 | |
---|
418 | if isinstance(o, dict): |
---|
419 | for k, i in o.items(): |
---|
420 | st.append(k) |
---|
421 | find_pickle_problem(i, st) |
---|
422 | st.pop() |
---|
423 | elif isinstance(o, list): |
---|
424 | st.append('list') |
---|
425 | for i in o: |
---|
426 | find_pickle_problem(i, st) |
---|
427 | st.pop() |
---|
428 | else: |
---|
429 | try: |
---|
430 | pickle.dumps(o) |
---|
431 | except pickle.PicklingError, e: |
---|
432 | print >>sys.stderr, "<PicklingError>" |
---|
433 | print >>sys.stderr, st |
---|
434 | print >>sys.stderr, o |
---|
435 | print >>sys.stderr, e |
---|
436 | print >>sys.stderr, "</PicklingError>" |
---|
437 | |
---|