1 | #!/usr/local/bin/python |
---|
2 | |
---|
3 | import re |
---|
4 | import os |
---|
5 | import string |
---|
6 | import logging |
---|
7 | import pickle |
---|
8 | |
---|
9 | import httplib |
---|
10 | |
---|
11 | from optparse import OptionParser |
---|
12 | |
---|
13 | from socket import sslerror |
---|
14 | from tempfile import mkstemp |
---|
15 | |
---|
16 | from M2Crypto import SSL |
---|
17 | from M2Crypto.SSL import SSLError |
---|
18 | from deter import fedid |
---|
19 | from service_error import service_error |
---|
20 | from urlparse import urlparse |
---|
21 | from M2Crypto import m2 |
---|
22 | |
---|
23 | |
---|
24 | # If this is an old enough version of M2Crypto.SSL that has an |
---|
25 | # ssl_verify_callback that doesn't allow 0-length signed certs, create a |
---|
26 | # version of that callback that does. This is edited from the original in |
---|
27 | # M2Crypto.SSL.cb. This version also elides the printing to stderr. |
---|
28 | if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None): |
---|
29 | from M2Crypto.SSL.Context import map |
---|
30 | |
---|
31 | def fedd_ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok): |
---|
32 | unknown_issuer = [ |
---|
33 | m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY, |
---|
34 | m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE, |
---|
35 | m2.X509_V_ERR_CERT_UNTRUSTED, |
---|
36 | m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT |
---|
37 | ] |
---|
38 | # m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed |
---|
39 | if getattr(m2, 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None): |
---|
40 | unknown_issuer.append(getattr(m2, |
---|
41 | 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None)) |
---|
42 | ssl_ctx = map()[ssl_ctx_ptr] |
---|
43 | |
---|
44 | if errnum in unknown_issuer: |
---|
45 | if ssl_ctx.get_allow_unknown_ca(): |
---|
46 | ok = 1 |
---|
47 | # CRL checking goes here... |
---|
48 | if ok: |
---|
49 | if ssl_ctx.get_verify_depth() >= errdepth: |
---|
50 | ok = 1 |
---|
51 | else: |
---|
52 | ok = 0 |
---|
53 | return ok |
---|
54 | else: |
---|
55 | def fedd_ssl_verify_callback(ok, store): |
---|
56 | ''' |
---|
57 | m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed |
---|
58 | ''' |
---|
59 | errnum = store.get_error() |
---|
60 | if errnum == m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN: |
---|
61 | ok = 1 |
---|
62 | return ok |
---|
63 | else: |
---|
64 | return SSL.cb.ssl_verify_callback_allow_unknown_ca(ok, store) |
---|
65 | |
---|
66 | class fedd_ssl_context(SSL.Context): |
---|
67 | """ |
---|
68 | Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd. |
---|
69 | """ |
---|
70 | def __init__(self, my_cert, trusted_certs=None, password=None, protocol=None): |
---|
71 | """ |
---|
72 | construct a fedd_ssl_context |
---|
73 | |
---|
74 | @param my_cert: PEM file with my certificate in it |
---|
75 | @param trusted_certs: PEM file with trusted certs in it (optional) |
---|
76 | """ |
---|
77 | if protocol: SSL.Context.__init__(self, protocol=protocol) |
---|
78 | else: SSL.Context.__init__(self) |
---|
79 | |
---|
80 | # load_cert takes a callback to get a password, not a password, so if |
---|
81 | # the caller provided a password, this creates a nonce callback using a |
---|
82 | # lambda form. |
---|
83 | if password != None and not callable(password): |
---|
84 | # This is cute. password = lambda *args: password produces a |
---|
85 | # function object that returns itself rather than one that returns |
---|
86 | # the object itself. This is because password is an object |
---|
87 | # reference and after the assignment it's a lambda. So we assign |
---|
88 | # to a temp. |
---|
89 | pwd = str(password) |
---|
90 | password =lambda *args: pwd |
---|
91 | |
---|
92 | # The calls to str below (and above) are because the underlying SSL |
---|
93 | # stuff is intolerant of unicode. |
---|
94 | if password != None: |
---|
95 | self.load_cert(str(my_cert), callback=password) |
---|
96 | else: |
---|
97 | self.load_cert(str(my_cert)) |
---|
98 | |
---|
99 | # If no trusted certificates are specified, allow unknown CAs. |
---|
100 | if trusted_certs: |
---|
101 | self.load_verify_locations(trusted_certs) |
---|
102 | self.set_verify(SSL.verify_peer, 10) |
---|
103 | else: |
---|
104 | # Install the proper callback to allow self-signed certs |
---|
105 | self.set_allow_unknown_ca(True) |
---|
106 | self.set_verify(SSL.verify_peer, 10, |
---|
107 | callback=fedd_ssl_verify_callback) |
---|
108 | |
---|
109 | # no session caching |
---|
110 | self.set_session_cache_mode(0) |
---|
111 | |
---|
112 | class file_expanding_opts(OptionParser): |
---|
113 | def expand_file(self, option, opt_str, v, p): |
---|
114 | """ |
---|
115 | Store the given value to the given destination after expanding home |
---|
116 | directories. |
---|
117 | """ |
---|
118 | setattr(p.values, option.dest, os.path.expanduser(v)) |
---|
119 | |
---|
120 | def __init__(self, usage=None, version=None): |
---|
121 | OptionParser.__init__(self) |
---|
122 | |
---|
123 | |
---|
124 | def read_simple_accessdb(fn, auth, mask=[]): |
---|
125 | """ |
---|
126 | Read a simple access database. Each line is a fedid (of the form |
---|
127 | fedid:hexstring) and a comma separated list of atributes to be assigned to |
---|
128 | it. This parses out the fedids and adds the attributes to the authorizer. |
---|
129 | comments (preceded with a #) and blank lines are ignored. Exceptions (e.g. |
---|
130 | file exceptions and ValueErrors from badly parsed lines) are propagated. |
---|
131 | """ |
---|
132 | |
---|
133 | rv = [ ] |
---|
134 | lineno = 0 |
---|
135 | fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\ |
---|
136 | "(\w+\s*(,\s*\w+)*)") |
---|
137 | |
---|
138 | # If a single string came in, make it a list |
---|
139 | if isinstance(mask, basestring): mask = [ mask ] |
---|
140 | |
---|
141 | f = open(fn, 'r') |
---|
142 | for line in f: |
---|
143 | lineno += 1 |
---|
144 | line = line.strip() |
---|
145 | if line.startswith('#') or len(line) == 0: |
---|
146 | continue |
---|
147 | m = fedid_line.match(line) |
---|
148 | if m : |
---|
149 | fid = fedid(hexstr=m.group(1)) |
---|
150 | for a in [ a.strip() for a in m.group(2).split(",") \ |
---|
151 | if not mask or a.strip() in mask ]: |
---|
152 | auth.set_attribute(fid, a.strip()) |
---|
153 | else: |
---|
154 | raise ValueError("Badly formatted line in accessdb: %s line %d" %\ |
---|
155 | (fn, lineno)) |
---|
156 | f.close() |
---|
157 | return rv |
---|
158 | |
---|
159 | |
---|
160 | def pack_id(id): |
---|
161 | """ |
---|
162 | Return a dictionary with the field name set by the id type. Handy for |
---|
163 | creating dictionaries to be converted to messages. |
---|
164 | """ |
---|
165 | if isinstance(id, fedid): return { 'fedid': id } |
---|
166 | elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id } |
---|
167 | else: return { 'localname': id} |
---|
168 | |
---|
169 | def unpack_id(id): |
---|
170 | """return id as a type determined by the key""" |
---|
171 | for k in ("localname", "fedid", "uri", "kerberosUsername"): |
---|
172 | if id.has_key(k): return id[k] |
---|
173 | return None |
---|
174 | |
---|
175 | def set_log_level(config, sect, log): |
---|
176 | """ Set the logging level to the value passed in sect of config.""" |
---|
177 | # The getattr sleight of hand finds the logging level constant |
---|
178 | # corrersponding to the string. We're a little paranoid to avoid user |
---|
179 | # mayhem. |
---|
180 | if config.has_option(sect, "log_level"): |
---|
181 | level_str = config.get(sect, "log_level") |
---|
182 | try: |
---|
183 | level = int(getattr(logging, level_str.upper(), -1)) |
---|
184 | |
---|
185 | if logging.DEBUG <= level <= logging.CRITICAL: |
---|
186 | log.setLevel(level) |
---|
187 | else: |
---|
188 | log.error("Bad experiment_log value: %s" % level_str) |
---|
189 | |
---|
190 | except ValueError: |
---|
191 | log.error("Bad experiment_log value: %s" % level_str) |
---|
192 | |
---|
193 | def copy_file(src, dest, size=1024): |
---|
194 | """ |
---|
195 | Exceedingly simple file copy. Throws an EnvironmentError if there's a problem. |
---|
196 | """ |
---|
197 | s = open(src,'r') |
---|
198 | d = open(dest, 'w') |
---|
199 | |
---|
200 | buf = s.read(size) |
---|
201 | while buf != "": |
---|
202 | d.write(buf) |
---|
203 | buf = s.read(size) |
---|
204 | s.close() |
---|
205 | d.close() |
---|
206 | |
---|
207 | def get_url(url, cf, destdir, fn=None, max_retries=5, log=None): |
---|
208 | """ |
---|
209 | Get data from a federated data store. This presents the client cert/fedid |
---|
210 | to the http server. We retry up to max_retries times. |
---|
211 | """ |
---|
212 | po = urlparse(url) |
---|
213 | if not fn: |
---|
214 | fn = po.path.rpartition('/')[2] |
---|
215 | retries = 0 |
---|
216 | ok = False |
---|
217 | failed_exception = None |
---|
218 | while not ok and retries < max_retries: |
---|
219 | try: |
---|
220 | if log: |
---|
221 | log.debug('Creating HTTPSConnection: %s' % cf) |
---|
222 | conn = httplib.HTTPSConnection(po.hostname, port=po.port, |
---|
223 | cert_file=cf, key_file=cf, timeout=30) |
---|
224 | conn.putrequest('GET', po.path) |
---|
225 | conn.endheaders() |
---|
226 | if log: |
---|
227 | log.debug('Connecting') |
---|
228 | response = conn.getresponse() |
---|
229 | if log: |
---|
230 | log.debug('Connected') |
---|
231 | |
---|
232 | lf = open("%s/%s" % (destdir, fn), "w") |
---|
233 | if log: |
---|
234 | log.debug('initial read') |
---|
235 | buf = response.read(4096) |
---|
236 | while buf: |
---|
237 | lf.write(buf) |
---|
238 | if log: |
---|
239 | log.debug('loop read') |
---|
240 | buf = response.read(4096) |
---|
241 | lf.close() |
---|
242 | ok = True |
---|
243 | except EnvironmentError, e: |
---|
244 | failed_exception = e |
---|
245 | retries += 1 |
---|
246 | except httplib.HTTPException, e: |
---|
247 | failed_exception = e |
---|
248 | retries += 1 |
---|
249 | except sslerror, e: |
---|
250 | failed_exception = e |
---|
251 | retries += 1 |
---|
252 | except SSLError, e: |
---|
253 | failed_exception = e |
---|
254 | retries += 1 |
---|
255 | |
---|
256 | if retries > max_retries and failed_exception: |
---|
257 | if log: |
---|
258 | log.debug('Raising %s', failed_exception) |
---|
259 | raise failed_excpetion |
---|
260 | |
---|
261 | # Functions to manipulate composite testbed names |
---|
262 | def testbed_base(tb): |
---|
263 | """ |
---|
264 | Simple code to get the base testebd name. |
---|
265 | """ |
---|
266 | i = tb.find('/') |
---|
267 | if i == -1: return tb |
---|
268 | else: return tb[0:i] |
---|
269 | |
---|
270 | def testbed_suffix(tb): |
---|
271 | """ |
---|
272 | Simple code to get a testbed suffix, if nay. No suffix returns None. |
---|
273 | """ |
---|
274 | i = tb.find('/') |
---|
275 | if i != -1: return tb[i+1:] |
---|
276 | else: return None |
---|
277 | |
---|
278 | def split_testbed(tb): |
---|
279 | """ |
---|
280 | Return a testbed and a suffix as a tuple. No suffix returns None for that |
---|
281 | field |
---|
282 | """ |
---|
283 | |
---|
284 | i = tb.find('/') |
---|
285 | if i != -1: return (tb[0:i], tb[i+1:]) |
---|
286 | else: return (tb, None) |
---|
287 | |
---|
288 | def join_testbed(base, suffix=None): |
---|
289 | """ |
---|
290 | Build a testbed with suffix. If base is itself a tuple, combine them, |
---|
291 | otherwise combine the two. |
---|
292 | """ |
---|
293 | if isinstance(base, tuple): |
---|
294 | if len(base) == 2: |
---|
295 | return '/'.join(base) |
---|
296 | else: |
---|
297 | raise RuntimeError("Too many tuple elements for join_testbed") |
---|
298 | else: |
---|
299 | if suffix: |
---|
300 | return '/'.join((base, suffix)) |
---|
301 | else: |
---|
302 | return base |
---|
303 | |
---|
304 | def abac_pem_type(cert): |
---|
305 | key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$') |
---|
306 | cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') |
---|
307 | type = None |
---|
308 | |
---|
309 | f = open(cert, 'r') |
---|
310 | for line in f: |
---|
311 | if key_re.match(line): |
---|
312 | if type is None: type = 'key' |
---|
313 | elif type == 'cert': type = 'both' |
---|
314 | elif cert_re.match(line): |
---|
315 | if type is None: type = 'cert' |
---|
316 | elif type == 'key': type = 'both' |
---|
317 | if type == 'both': break |
---|
318 | f.close() |
---|
319 | return type |
---|
320 | |
---|
321 | def abac_split_cert(cert, keyfile=None, certfile=None): |
---|
322 | """ |
---|
323 | Split the certificate file in cert into a certificate file and a key file |
---|
324 | in cf and kf respectively. The ABAC tools generally cannot handle combined |
---|
325 | certificates/keys. If kf anc cf are given, they are used, otherwise tmp |
---|
326 | files are created. Created tmp files must be deleted. Problems opening or |
---|
327 | writing files will cause exceptions. |
---|
328 | """ |
---|
329 | class diversion: |
---|
330 | ''' |
---|
331 | Wraps up the reqular expression to start and end a diversion, as well as |
---|
332 | the open file that gets the lines. If fd is passed in, use that system |
---|
333 | file (probably from a mkstemp. Otherwise open the given filename. |
---|
334 | ''' |
---|
335 | def __init__(self, start, end, fn=None, fd=None): |
---|
336 | self.start = re.compile(start) |
---|
337 | self.end = re.compile(end) |
---|
338 | |
---|
339 | if not fd: |
---|
340 | # Open the file securely with minimal permissions. NB file |
---|
341 | # cannot exist before this call. |
---|
342 | fd = os.open(fn, |
---|
343 | (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600) |
---|
344 | |
---|
345 | self.f = os.fdopen(fd, 'w') |
---|
346 | |
---|
347 | def close(self): |
---|
348 | self.f.close() |
---|
349 | |
---|
350 | if not keyfile: |
---|
351 | kf, rkeyfile = mkstemp(suffix=".pem") |
---|
352 | else: |
---|
353 | kf, rkeyfile = None, keyfile |
---|
354 | |
---|
355 | if not certfile: |
---|
356 | cf, rcertfile = mkstemp(suffix=".pem") |
---|
357 | else: |
---|
358 | cf, rcertfile = None, certfile |
---|
359 | |
---|
360 | # Initialize the diversions |
---|
361 | divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in ( |
---|
362 | ('\s*-----BEGIN RSA PRIVATE KEY-----$', |
---|
363 | '\s*-----END RSA PRIVATE KEY-----$', |
---|
364 | keyfile, kf), |
---|
365 | ('\s*-----BEGIN CERTIFICATE-----$', |
---|
366 | '\s*-----END CERTIFICATE-----$', |
---|
367 | certfile, cf))] |
---|
368 | |
---|
369 | # walk through the file, beginning a diversion when a start regexp |
---|
370 | # matches until the end regexp matches. While in the two regexps, |
---|
371 | # print each line to the open diversion file (including the two |
---|
372 | # matches). |
---|
373 | active = None |
---|
374 | f = open(cert, 'r') |
---|
375 | for l in f: |
---|
376 | if active: |
---|
377 | if active.end.match(l): |
---|
378 | print >>active.f, l, |
---|
379 | active = None |
---|
380 | else: |
---|
381 | for d in divs: |
---|
382 | if d.start.match(l): |
---|
383 | active = d |
---|
384 | break |
---|
385 | if active: print >>active.f, l, |
---|
386 | |
---|
387 | # This is probably unnecessary. Close all the diversion files. |
---|
388 | for d in divs: d.close() |
---|
389 | return rkeyfile, rcertfile |
---|
390 | |
---|
391 | def abac_context_to_creds(context): |
---|
392 | """ |
---|
393 | Pull all the credentials out of the context and return 2 lists of the |
---|
394 | underlying credentials in an exportable format, IDs and attributes. |
---|
395 | There are no duplicates in the lists. |
---|
396 | """ |
---|
397 | ids, attrs = set(), set() |
---|
398 | # This should be a one-iteration loop |
---|
399 | for c in context.credentials(): |
---|
400 | ids.add(str(c.issuer_cert())) |
---|
401 | attrs.add(str(c.attribute_cert())) |
---|
402 | |
---|
403 | return list(ids), list(attrs) |
---|
404 | |
---|
405 | def find_pickle_problem(o, st=None): |
---|
406 | """ |
---|
407 | Debugging routine to figure out what doesn't pickle from a dict full of |
---|
408 | dicts and lists. It tries to walk down the lists and dicts and pickle each |
---|
409 | atom. If something fails to pickle, it prints an approximation of a stack |
---|
410 | trace through the data structure. |
---|
411 | """ |
---|
412 | if st is None: st = [ ] |
---|
413 | |
---|
414 | if isinstance(o, dict): |
---|
415 | for k, i in o.items(): |
---|
416 | st.append(k) |
---|
417 | find_pickle_problem(i, st) |
---|
418 | st.pop() |
---|
419 | elif isinstance(o, list): |
---|
420 | st.append('list') |
---|
421 | for i in o: |
---|
422 | find_pickle_problem(i, st) |
---|
423 | st.pop() |
---|
424 | else: |
---|
425 | try: |
---|
426 | pickle.dumps(o) |
---|
427 | except pickle.PicklingError, e: |
---|
428 | print >>sys.stderr, "<PicklingError>" |
---|
429 | print >>sys.stderr, st |
---|
430 | print >>sys.stderr, o |
---|
431 | print >>sys.stderr, e |
---|
432 | print >>sys.stderr, "</PicklingError>" |
---|
433 | |
---|