[6ff0b91] | 1 | #!/usr/local/bin/python |
---|
| 2 | |
---|
[05191a6] | 3 | import re |
---|
[353db8c] | 4 | import os |
---|
[05191a6] | 5 | import string |
---|
[2c6128f] | 6 | import logging |
---|
[b53e1fc] | 7 | import pickle |
---|
[6ff0b91] | 8 | |
---|
[1dcaff4] | 9 | import httplib |
---|
| 10 | |
---|
[62f3dd9] | 11 | from optparse import OptionParser |
---|
| 12 | |
---|
[10a7053] | 13 | from socket import sslerror |
---|
[6e63513] | 14 | from tempfile import mkstemp |
---|
[10a7053] | 15 | |
---|
[51cc9df] | 16 | from M2Crypto import SSL |
---|
[1dcaff4] | 17 | from M2Crypto.SSL import SSLError |
---|
[6bedbdba] | 18 | from deter import fedid |
---|
[1dcaff4] | 19 | from service_error import service_error |
---|
| 20 | from urlparse import urlparse |
---|
[816daef] | 21 | from M2Crypto import m2 |
---|
[6ff0b91] | 22 | |
---|
[fe157b9] | 23 | |
---|
| 24 | # If this is an old enough version of M2Crypto.SSL that has an |
---|
| 25 | # ssl_verify_callback that doesn't allow 0-length signed certs, create a |
---|
| 26 | # version of that callback that does. This is edited from the original in |
---|
| 27 | # M2Crypto.SSL.cb. This version also elides the printing to stderr. |
---|
| 28 | if not getattr(SSL.cb, 'ssl_verify_callback_allow_unknown_ca', None): |
---|
| 29 | from M2Crypto.SSL.Context import map |
---|
| 30 | |
---|
[816daef] | 31 | def fedd_ssl_verify_callback(ssl_ctx_ptr, x509_ptr, errnum, errdepth, ok): |
---|
[fe157b9] | 32 | unknown_issuer = [ |
---|
| 33 | m2.X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY, |
---|
| 34 | m2.X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE, |
---|
| 35 | m2.X509_V_ERR_CERT_UNTRUSTED, |
---|
| 36 | m2.X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT |
---|
| 37 | ] |
---|
[816daef] | 38 | # m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed |
---|
| 39 | if getattr(m2, 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None): |
---|
| 40 | unknown_issuer.append(getattr(m2, |
---|
| 41 | 'X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN', None)) |
---|
[fe157b9] | 42 | ssl_ctx = map()[ssl_ctx_ptr] |
---|
| 43 | |
---|
| 44 | if errnum in unknown_issuer: |
---|
| 45 | if ssl_ctx.get_allow_unknown_ca(): |
---|
| 46 | ok = 1 |
---|
| 47 | # CRL checking goes here... |
---|
| 48 | if ok: |
---|
| 49 | if ssl_ctx.get_verify_depth() >= errdepth: |
---|
| 50 | ok = 1 |
---|
| 51 | else: |
---|
| 52 | ok = 0 |
---|
| 53 | return ok |
---|
| 54 | else: |
---|
[816daef] | 55 | def fedd_ssl_verify_callback(ok, store): |
---|
| 56 | ''' |
---|
| 57 | m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN should also be allowed |
---|
| 58 | ''' |
---|
| 59 | errnum = store.get_error() |
---|
| 60 | if errnum == m2.X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN: |
---|
| 61 | ok = 1 |
---|
| 62 | return ok |
---|
| 63 | else: |
---|
| 64 | return SSL.cb.ssl_verify_callback_allow_unknown_ca(ok, store) |
---|
[fe157b9] | 65 | |
---|
[6ff0b91] | 66 | class fedd_ssl_context(SSL.Context): |
---|
| 67 | """ |
---|
| 68 | Simple wrapper around an M2Crypto.SSL.Context to initialize it for fedd. |
---|
| 69 | """ |
---|
[4241f3c] | 70 | def __init__(self, my_cert, trusted_certs=None, password=None, protocol=None): |
---|
[6ff0b91] | 71 | """ |
---|
| 72 | construct a fedd_ssl_context |
---|
| 73 | |
---|
| 74 | @param my_cert: PEM file with my certificate in it |
---|
| 75 | @param trusted_certs: PEM file with trusted certs in it (optional) |
---|
| 76 | """ |
---|
[4241f3c] | 77 | if protocol: SSL.Context.__init__(self, protocol=protocol) |
---|
| 78 | else: SSL.Context.__init__(self) |
---|
[6ff0b91] | 79 | |
---|
| 80 | # load_cert takes a callback to get a password, not a password, so if |
---|
| 81 | # the caller provided a password, this creates a nonce callback using a |
---|
| 82 | # lambda form. |
---|
| 83 | if password != None and not callable(password): |
---|
| 84 | # This is cute. password = lambda *args: password produces a |
---|
| 85 | # function object that returns itself rather than one that returns |
---|
| 86 | # the object itself. This is because password is an object |
---|
| 87 | # reference and after the assignment it's a lambda. So we assign |
---|
| 88 | # to a temp. |
---|
[632dd59] | 89 | pwd = str(password) |
---|
[6ff0b91] | 90 | password =lambda *args: pwd |
---|
| 91 | |
---|
[632dd59] | 92 | # The calls to str below (and above) are because the underlying SSL |
---|
| 93 | # stuff is intolerant of unicode. |
---|
[6ff0b91] | 94 | if password != None: |
---|
[632dd59] | 95 | self.load_cert(str(my_cert), callback=password) |
---|
[6ff0b91] | 96 | else: |
---|
[632dd59] | 97 | self.load_cert(str(my_cert)) |
---|
[f069052] | 98 | |
---|
| 99 | # If no trusted certificates are specified, allow unknown CAs. |
---|
| 100 | if trusted_certs: |
---|
| 101 | self.load_verify_locations(trusted_certs) |
---|
| 102 | self.set_verify(SSL.verify_peer, 10) |
---|
| 103 | else: |
---|
[816daef] | 104 | # Install the proper callback to allow self-signed certs |
---|
[12e6ca8] | 105 | self.set_allow_unknown_ca(True) |
---|
[816daef] | 106 | self.set_verify(SSL.verify_peer, 10, |
---|
| 107 | callback=fedd_ssl_verify_callback) |
---|
[6ff0b91] | 108 | |
---|
[0dc62df] | 109 | # no session caching |
---|
| 110 | self.set_session_cache_mode(0) |
---|
| 111 | |
---|
[62f3dd9] | 112 | class file_expanding_opts(OptionParser): |
---|
| 113 | def expand_file(self, option, opt_str, v, p): |
---|
| 114 | """ |
---|
| 115 | Store the given value to the given destination after expanding home |
---|
| 116 | directories. |
---|
| 117 | """ |
---|
| 118 | setattr(p.values, option.dest, os.path.expanduser(v)) |
---|
| 119 | |
---|
| 120 | def __init__(self, usage=None, version=None): |
---|
| 121 | OptionParser.__init__(self) |
---|
| 122 | |
---|
| 123 | |
---|
[05191a6] | 124 | def read_simple_accessdb(fn, auth, mask=[]): |
---|
| 125 | """ |
---|
| 126 | Read a simple access database. Each line is a fedid (of the form |
---|
| 127 | fedid:hexstring) and a comma separated list of atributes to be assigned to |
---|
| 128 | it. This parses out the fedids and adds the attributes to the authorizer. |
---|
| 129 | comments (preceded with a #) and blank lines are ignored. Exceptions (e.g. |
---|
| 130 | file exceptions and ValueErrors from badly parsed lines) are propagated. |
---|
| 131 | """ |
---|
| 132 | |
---|
| 133 | rv = [ ] |
---|
| 134 | lineno = 0 |
---|
| 135 | fedid_line = re.compile("fedid:([" + string.hexdigits + "]+)\s+" +\ |
---|
| 136 | "(\w+\s*(,\s*\w+)*)") |
---|
| 137 | |
---|
| 138 | # If a single string came in, make it a list |
---|
| 139 | if isinstance(mask, basestring): mask = [ mask ] |
---|
| 140 | |
---|
| 141 | f = open(fn, 'r') |
---|
| 142 | for line in f: |
---|
| 143 | lineno += 1 |
---|
| 144 | line = line.strip() |
---|
| 145 | if line.startswith('#') or len(line) == 0: |
---|
| 146 | continue |
---|
| 147 | m = fedid_line.match(line) |
---|
| 148 | if m : |
---|
| 149 | fid = fedid(hexstr=m.group(1)) |
---|
| 150 | for a in [ a.strip() for a in m.group(2).split(",") \ |
---|
| 151 | if not mask or a.strip() in mask ]: |
---|
| 152 | auth.set_attribute(fid, a.strip()) |
---|
| 153 | else: |
---|
| 154 | raise ValueError("Badly formatted line in accessdb: %s line %d" %\ |
---|
[cc8d8e9] | 155 | (fn, lineno)) |
---|
[05191a6] | 156 | f.close() |
---|
| 157 | return rv |
---|
| 158 | |
---|
| 159 | |
---|
[6ff0b91] | 160 | def pack_id(id): |
---|
| 161 | """ |
---|
[3e293e4] | 162 | Return a dictionary with the field name set by the id type. Handy for |
---|
| 163 | creating dictionaries to be converted to messages. |
---|
[6ff0b91] | 164 | """ |
---|
[f8b118e] | 165 | if isinstance(id, fedid): return { 'fedid': id } |
---|
[6ff0b91] | 166 | elif id.startswith("http:") or id.startswith("https:"): return { 'uri': id } |
---|
[e40c7ee] | 167 | else: return { 'localname': id} |
---|
[6ff0b91] | 168 | |
---|
[2106ed1] | 169 | def unpack_id(id): |
---|
| 170 | """return id as a type determined by the key""" |
---|
[f8b118e] | 171 | for k in ("localname", "fedid", "uri", "kerberosUsername"): |
---|
| 172 | if id.has_key(k): return id[k] |
---|
[2106ed1] | 173 | return None |
---|
| 174 | |
---|
[2c6128f] | 175 | def set_log_level(config, sect, log): |
---|
| 176 | """ Set the logging level to the value passed in sect of config.""" |
---|
| 177 | # The getattr sleight of hand finds the logging level constant |
---|
| 178 | # corrersponding to the string. We're a little paranoid to avoid user |
---|
| 179 | # mayhem. |
---|
| 180 | if config.has_option(sect, "log_level"): |
---|
| 181 | level_str = config.get(sect, "log_level") |
---|
| 182 | try: |
---|
| 183 | level = int(getattr(logging, level_str.upper(), -1)) |
---|
| 184 | |
---|
| 185 | if logging.DEBUG <= level <= logging.CRITICAL: |
---|
| 186 | log.setLevel(level) |
---|
| 187 | else: |
---|
| 188 | log.error("Bad experiment_log value: %s" % level_str) |
---|
| 189 | |
---|
| 190 | except ValueError: |
---|
| 191 | log.error("Bad experiment_log value: %s" % level_str) |
---|
| 192 | |
---|
[40dd8c1] | 193 | def copy_file(src, dest, size=1024): |
---|
| 194 | """ |
---|
[d3c8759] | 195 | Exceedingly simple file copy. Throws an EnvironmentError if there's a problem. |
---|
[40dd8c1] | 196 | """ |
---|
| 197 | s = open(src,'r') |
---|
| 198 | d = open(dest, 'w') |
---|
| 199 | |
---|
| 200 | buf = s.read(size) |
---|
| 201 | while buf != "": |
---|
| 202 | d.write(buf) |
---|
| 203 | buf = s.read(size) |
---|
| 204 | s.close() |
---|
| 205 | d.close() |
---|
| 206 | |
---|
[815cd26] | 207 | def get_url(url, cf, destdir, fn=None, max_retries=5, log=None): |
---|
[1dcaff4] | 208 | """ |
---|
| 209 | Get data from a federated data store. This presents the client cert/fedid |
---|
| 210 | to the http server. We retry up to max_retries times. |
---|
| 211 | """ |
---|
| 212 | po = urlparse(url) |
---|
| 213 | if not fn: |
---|
| 214 | fn = po.path.rpartition('/')[2] |
---|
| 215 | retries = 0 |
---|
| 216 | ok = False |
---|
| 217 | failed_exception = None |
---|
[f490c75] | 218 | while not ok and retries < max_retries: |
---|
[1dcaff4] | 219 | try: |
---|
| 220 | conn = httplib.HTTPSConnection(po.hostname, port=po.port, |
---|
[815cd26] | 221 | cert_file=cf, key_file=cf, timeout=30) |
---|
[1dcaff4] | 222 | conn.putrequest('GET', po.path) |
---|
| 223 | conn.endheaders() |
---|
| 224 | response = conn.getresponse() |
---|
| 225 | |
---|
| 226 | lf = open("%s/%s" % (destdir, fn), "w") |
---|
| 227 | buf = response.read(4096) |
---|
| 228 | while buf: |
---|
| 229 | lf.write(buf) |
---|
| 230 | buf = response.read(4096) |
---|
| 231 | lf.close() |
---|
| 232 | ok = True |
---|
[d3c8759] | 233 | except EnvironmentError, e: |
---|
[9f4b3c6] | 234 | failed_exception = e |
---|
[1dcaff4] | 235 | retries += 1 |
---|
| 236 | except httplib.HTTPException, e: |
---|
| 237 | failed_exception = e |
---|
| 238 | retries += 1 |
---|
[10a7053] | 239 | except sslerror, e: |
---|
| 240 | failed_exception = e |
---|
| 241 | retries += 1 |
---|
[1dcaff4] | 242 | except SSLError, e: |
---|
| 243 | failed_exception = e |
---|
| 244 | retries += 1 |
---|
| 245 | |
---|
[f490c75] | 246 | if retries > max_retries and failed_exception: |
---|
[9f4b3c6] | 247 | if log: |
---|
| 248 | log.debug('Raising %s', failed_exception) |
---|
[1dcaff4] | 249 | raise failed_excpetion |
---|
| 250 | |
---|
[ab847bc] | 251 | # Functions to manipulate composite testbed names |
---|
| 252 | def testbed_base(tb): |
---|
| 253 | """ |
---|
| 254 | Simple code to get the base testebd name. |
---|
| 255 | """ |
---|
| 256 | i = tb.find('/') |
---|
| 257 | if i == -1: return tb |
---|
| 258 | else: return tb[0:i] |
---|
| 259 | |
---|
| 260 | def testbed_suffix(tb): |
---|
| 261 | """ |
---|
[19d2b72] | 262 | Simple code to get a testbed suffix, if any. No suffix returns None. |
---|
[ab847bc] | 263 | """ |
---|
| 264 | i = tb.find('/') |
---|
| 265 | if i != -1: return tb[i+1:] |
---|
| 266 | else: return None |
---|
| 267 | |
---|
| 268 | def split_testbed(tb): |
---|
| 269 | """ |
---|
| 270 | Return a testbed and a suffix as a tuple. No suffix returns None for that |
---|
| 271 | field |
---|
| 272 | """ |
---|
| 273 | |
---|
| 274 | i = tb.find('/') |
---|
| 275 | if i != -1: return (tb[0:i], tb[i+1:]) |
---|
| 276 | else: return (tb, None) |
---|
| 277 | |
---|
| 278 | def join_testbed(base, suffix=None): |
---|
| 279 | """ |
---|
| 280 | Build a testbed with suffix. If base is itself a tuple, combine them, |
---|
| 281 | otherwise combine the two. |
---|
| 282 | """ |
---|
| 283 | if isinstance(base, tuple): |
---|
| 284 | if len(base) == 2: |
---|
| 285 | return '/'.join(base) |
---|
| 286 | else: |
---|
| 287 | raise RuntimeError("Too many tuple elements for join_testbed") |
---|
| 288 | else: |
---|
| 289 | if suffix: |
---|
| 290 | return '/'.join((base, suffix)) |
---|
| 291 | else: |
---|
| 292 | return base |
---|
[b53e1fc] | 293 | |
---|
[353db8c] | 294 | def abac_pem_type(cert): |
---|
| 295 | key_re = re.compile('\s*-----BEGIN RSA PRIVATE KEY-----$') |
---|
| 296 | cert_re = re.compile('\s*-----BEGIN CERTIFICATE-----$') |
---|
| 297 | type = None |
---|
[e62245e] | 298 | |
---|
[353db8c] | 299 | f = open(cert, 'r') |
---|
| 300 | for line in f: |
---|
| 301 | if key_re.match(line): |
---|
| 302 | if type is None: type = 'key' |
---|
| 303 | elif type == 'cert': type = 'both' |
---|
| 304 | elif cert_re.match(line): |
---|
| 305 | if type is None: type = 'cert' |
---|
| 306 | elif type == 'key': type = 'both' |
---|
| 307 | if type == 'both': break |
---|
| 308 | f.close() |
---|
| 309 | return type |
---|
| 310 | |
---|
| 311 | def abac_split_cert(cert, keyfile=None, certfile=None): |
---|
| 312 | """ |
---|
| 313 | Split the certificate file in cert into a certificate file and a key file |
---|
| 314 | in cf and kf respectively. The ABAC tools generally cannot handle combined |
---|
| 315 | certificates/keys. If kf anc cf are given, they are used, otherwise tmp |
---|
| 316 | files are created. Created tmp files must be deleted. Problems opening or |
---|
| 317 | writing files will cause exceptions. |
---|
| 318 | """ |
---|
| 319 | class diversion: |
---|
| 320 | ''' |
---|
| 321 | Wraps up the reqular expression to start and end a diversion, as well as |
---|
[6e63513] | 322 | the open file that gets the lines. If fd is passed in, use that system |
---|
| 323 | file (probably from a mkstemp. Otherwise open the given filename. |
---|
[353db8c] | 324 | ''' |
---|
[6e63513] | 325 | def __init__(self, start, end, fn=None, fd=None): |
---|
[353db8c] | 326 | self.start = re.compile(start) |
---|
| 327 | self.end = re.compile(end) |
---|
[6e63513] | 328 | |
---|
| 329 | if not fd: |
---|
| 330 | # Open the file securely with minimal permissions. NB file |
---|
| 331 | # cannot exist before this call. |
---|
| 332 | fd = os.open(fn, |
---|
| 333 | (os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_EXCL), 0600) |
---|
| 334 | |
---|
| 335 | self.f = os.fdopen(fd, 'w') |
---|
[353db8c] | 336 | |
---|
| 337 | def close(self): |
---|
| 338 | self.f.close() |
---|
| 339 | |
---|
| 340 | if not keyfile: |
---|
[6e63513] | 341 | kf, rkeyfile = mkstemp(suffix=".pem") |
---|
| 342 | else: |
---|
| 343 | kf, rkeyfile = None, keyfile |
---|
| 344 | |
---|
[353db8c] | 345 | if not certfile: |
---|
[6e63513] | 346 | cf, rcertfile = mkstemp(suffix=".pem") |
---|
| 347 | else: |
---|
| 348 | cf, rcertfile = None, certfile |
---|
[353db8c] | 349 | |
---|
| 350 | # Initialize the diversions |
---|
[6e63513] | 351 | divs = [diversion(s, e, fn=pfn, fd=pfd ) for s, e, pfn, pfd in ( |
---|
[353db8c] | 352 | ('\s*-----BEGIN RSA PRIVATE KEY-----$', |
---|
| 353 | '\s*-----END RSA PRIVATE KEY-----$', |
---|
[6e63513] | 354 | keyfile, kf), |
---|
[353db8c] | 355 | ('\s*-----BEGIN CERTIFICATE-----$', |
---|
| 356 | '\s*-----END CERTIFICATE-----$', |
---|
[6e63513] | 357 | certfile, cf))] |
---|
[353db8c] | 358 | |
---|
| 359 | # walk through the file, beginning a diversion when a start regexp |
---|
| 360 | # matches until the end regexp matches. While in the two regexps, |
---|
| 361 | # print each line to the open diversion file (including the two |
---|
| 362 | # matches). |
---|
| 363 | active = None |
---|
| 364 | f = open(cert, 'r') |
---|
| 365 | for l in f: |
---|
| 366 | if active: |
---|
| 367 | if active.end.match(l): |
---|
| 368 | print >>active.f, l, |
---|
| 369 | active = None |
---|
| 370 | else: |
---|
| 371 | for d in divs: |
---|
| 372 | if d.start.match(l): |
---|
| 373 | active = d |
---|
| 374 | break |
---|
| 375 | if active: print >>active.f, l, |
---|
| 376 | |
---|
| 377 | # This is probably unnecessary. Close all the diversion files. |
---|
| 378 | for d in divs: d.close() |
---|
[6e63513] | 379 | return rkeyfile, rcertfile |
---|
[353db8c] | 380 | |
---|
[c573278] | 381 | def abac_context_to_creds(context): |
---|
| 382 | """ |
---|
| 383 | Pull all the credentials out of the context and return 2 lists of the |
---|
| 384 | underlying credentials in an exportable format, IDs and attributes. |
---|
| 385 | There are no duplicates in the lists. |
---|
| 386 | """ |
---|
| 387 | ids, attrs = set(), set() |
---|
| 388 | # This should be a one-iteration loop |
---|
| 389 | for c in context.credentials(): |
---|
[cd5d16e] | 390 | ids.add(str(c.issuer_cert())) |
---|
| 391 | attrs.add(str(c.attribute_cert())) |
---|
[c573278] | 392 | |
---|
| 393 | return list(ids), list(attrs) |
---|
| 394 | |
---|
[b53e1fc] | 395 | def find_pickle_problem(o, st=None): |
---|
| 396 | """ |
---|
| 397 | Debugging routine to figure out what doesn't pickle from a dict full of |
---|
| 398 | dicts and lists. It tries to walk down the lists and dicts and pickle each |
---|
| 399 | atom. If something fails to pickle, it prints an approximation of a stack |
---|
| 400 | trace through the data structure. |
---|
| 401 | """ |
---|
| 402 | if st is None: st = [ ] |
---|
| 403 | |
---|
| 404 | if isinstance(o, dict): |
---|
| 405 | for k, i in o.items(): |
---|
| 406 | st.append(k) |
---|
| 407 | find_pickle_problem(i, st) |
---|
| 408 | st.pop() |
---|
| 409 | elif isinstance(o, list): |
---|
| 410 | st.append('list') |
---|
| 411 | for i in o: |
---|
| 412 | find_pickle_problem(i, st) |
---|
| 413 | st.pop() |
---|
| 414 | else: |
---|
| 415 | try: |
---|
| 416 | pickle.dumps(o) |
---|
| 417 | except pickle.PicklingError, e: |
---|
| 418 | print >>sys.stderr, "<PicklingError>" |
---|
| 419 | print >>sys.stderr, st |
---|
| 420 | print >>sys.stderr, o |
---|
| 421 | print >>sys.stderr, e |
---|
| 422 | print >>sys.stderr, "</PicklingError>" |
---|
| 423 | |
---|