[06c1dba] | 1 | #!/usr/local/bin/python |
---|
| 2 | |
---|
| 3 | import logging |
---|
| 4 | import util |
---|
[6bedbdba] | 5 | from deter import topdl |
---|
[06c1dba] | 6 | |
---|
| 7 | from M2Crypto import SSL |
---|
| 8 | from M2Crypto.m2xmlrpclib import SSL_Transport |
---|
| 9 | from xmlrpclib import ServerProxy, dumps, loads, Fault, Error, Binary |
---|
| 10 | from M2Crypto.SSL import SSLError |
---|
| 11 | from M2Crypto.BIO import BIOError |
---|
| 12 | from socket import error as socket_error |
---|
| 13 | from socket import sslerror |
---|
| 14 | import httplib |
---|
| 15 | from federation.util import fedd_ssl_context |
---|
| 16 | from federation import service_error |
---|
| 17 | from federation.operation_status import operation_status |
---|
| 18 | |
---|
| 19 | class xmlrpc_emulab_segment: |
---|
| 20 | class node_info: |
---|
| 21 | def __init__(self, name, pname, status=None, osname=None, |
---|
| 22 | osversion=None, image=None): |
---|
| 23 | self.name = name |
---|
| 24 | self.pname = pname |
---|
| 25 | self.status = status |
---|
| 26 | self.osname = osname |
---|
| 27 | self.osversion = osversion |
---|
| 28 | self.image = image |
---|
| 29 | def getOS(self): |
---|
| 30 | rv = None |
---|
| 31 | if self.osname or self.osversion: |
---|
| 32 | rv = topdl.OperatingSystem(name=self.osname, |
---|
| 33 | version=self.osversion) |
---|
| 34 | if self.image and rv: |
---|
| 35 | rv.set_attribute('emulab_access:image', self.image) |
---|
| 36 | return rv |
---|
| 37 | |
---|
[c7141dc] | 38 | def __init__(self, boss, ops, cert): |
---|
[06c1dba] | 39 | self.ctxt = fedd_ssl_context(my_cert=cert) |
---|
| 40 | self.ctxt.set_verify(SSL.verify_none, 10) |
---|
| 41 | self.boss = boss |
---|
[c7141dc] | 42 | self.ops = ops |
---|
[06c1dba] | 43 | self.null = """ |
---|
| 44 | set ns [new Simulator] |
---|
| 45 | source tb_compat.tcl |
---|
| 46 | |
---|
| 47 | set a [$ns node] |
---|
| 48 | |
---|
| 49 | $ns rtproto Session |
---|
| 50 | $ns run |
---|
| 51 | """ |
---|
| 52 | self.log = getattr(self, 'log', None) |
---|
| 53 | self.debug = getattr(self, 'debug', False) |
---|
| 54 | self.node = { } |
---|
| 55 | self.status = [ ] |
---|
[c7141dc] | 56 | self.node_info = xmlrpc_emulab_segment.node_info |
---|
[06c1dba] | 57 | |
---|
| 58 | def emulab_call(self, method, params): |
---|
| 59 | VERSION = 0.1 |
---|
| 60 | try: |
---|
| 61 | transport = SSL_Transport(self.ctxt) |
---|
| 62 | port = ServerProxy(self.boss, transport=transport) |
---|
| 63 | remote_method = getattr(port, method, None) |
---|
| 64 | if remote_method is not None: |
---|
| 65 | resp = remote_method(VERSION, params) |
---|
| 66 | else: |
---|
| 67 | raise service_error(service_error.internal, |
---|
| 68 | "Bad method: %s" % method) |
---|
| 69 | except socket_error, e: |
---|
| 70 | raise service_error(service_error.connect, |
---|
| 71 | "Cannot connect" % e) |
---|
| 72 | raise e |
---|
| 73 | except BIOError, e: |
---|
| 74 | if self.log: |
---|
| 75 | self.log.warn("BIO error: %s" % e) |
---|
| 76 | raise e |
---|
| 77 | except sslerror, e: |
---|
| 78 | if self.log: |
---|
| 79 | self.log.warn("SSL (socket) error: %s" % e) |
---|
| 80 | raise e |
---|
| 81 | except SSLError, e: |
---|
| 82 | if self.log: |
---|
| 83 | self.log.warn("SSL error: %s" % e) |
---|
| 84 | raise e |
---|
| 85 | except httplib.HTTPException, e: |
---|
| 86 | if self.log: |
---|
| 87 | self.log.warn("HTTP error: %s" % e) |
---|
| 88 | raise e |
---|
| 89 | except Fault, f: |
---|
| 90 | raise service_error(service_error.protocol, |
---|
| 91 | "Remote XMLRPC Fault: %s" % f) |
---|
| 92 | except Error, e: |
---|
| 93 | raise service_error(service_error.protocol, |
---|
| 94 | "Remote XMLRPC Fault: %s" % e) |
---|
| 95 | |
---|
| 96 | code = resp.get('code', -1) |
---|
| 97 | if code ==0: |
---|
| 98 | return (code, resp.get('value', None)) |
---|
| 99 | else: |
---|
| 100 | return (code, resp.get('output', None)) |
---|
| 101 | |
---|
| 102 | def get_state(self, pid, eid): |
---|
| 103 | """ |
---|
| 104 | Return the state of the experiment as reported by emulab |
---|
| 105 | """ |
---|
| 106 | |
---|
| 107 | if self.debug: |
---|
| 108 | state = 'swapped' |
---|
| 109 | else: |
---|
| 110 | params = { 'proj': pid, 'exp': eid } |
---|
| 111 | code, state = self.emulab_call('experiment.state', params) |
---|
| 112 | if code != 0: |
---|
| 113 | state = 'none' |
---|
| 114 | |
---|
| 115 | if self.log: |
---|
| 116 | self.log.debug("State is %s" % state) |
---|
| 117 | return state |
---|
| 118 | |
---|
| 119 | def make_null_experiment(self, pid, eid, tmpdir, gid=None): |
---|
| 120 | """ |
---|
| 121 | Create a null copy of the experiment so that we capture any logs there |
---|
| 122 | if the modify fails. Emulab software discards the logs from a failed |
---|
| 123 | startexp. |
---|
| 124 | """ |
---|
| 125 | |
---|
| 126 | if self.debug: |
---|
| 127 | if self.log: |
---|
| 128 | self.log.debug("[make_null_experiment]: " + \ |
---|
| 129 | "(debug) Creating experiment") |
---|
| 130 | return True |
---|
| 131 | else: |
---|
| 132 | params = { |
---|
| 133 | 'proj': pid, |
---|
| 134 | 'exp': eid, |
---|
| 135 | 'nsfilestr': self.null, |
---|
| 136 | 'batch': False, |
---|
| 137 | 'idleswap': 0, |
---|
| 138 | 'noidleswap_reason': 'Federated experiment', |
---|
| 139 | 'noswapin': True, |
---|
| 140 | 'wait': True |
---|
| 141 | } |
---|
| 142 | if gid is not None: |
---|
| 143 | params['group'] = gid |
---|
| 144 | if self.log: |
---|
| 145 | self.log.info("[make_null_experiment]: Creating experiment") |
---|
| 146 | code, value = self.emulab_call('experiment.startexp', params) |
---|
| 147 | |
---|
| 148 | if self.log: |
---|
| 149 | if code == 0: |
---|
| 150 | self.log.info('[make_null_experiment]: Create succeeded') |
---|
| 151 | else: |
---|
| 152 | self.log.error('[make_null_experiment]: Create failed: %s' \ |
---|
| 153 | % value) |
---|
| 154 | |
---|
| 155 | return code == 0 |
---|
| 156 | |
---|
| 157 | def swap_exp(self, pid, eid, direction='out', wait=True): |
---|
| 158 | """ |
---|
| 159 | Swap experiment in. |
---|
| 160 | """ |
---|
| 161 | if self.debug: |
---|
| 162 | if self.log: |
---|
| 163 | self.log.info("[swap_exp]: (debug) Swapping %s %s" % \ |
---|
| 164 | (eid, direction)) |
---|
| 165 | return True |
---|
| 166 | else: |
---|
| 167 | if self.log: |
---|
| 168 | self.log.info("[swap_exp]: Swapping %s %s" % (eid, direction)) |
---|
| 169 | params = { |
---|
| 170 | 'proj': pid, |
---|
| 171 | 'exp': eid, |
---|
| 172 | 'direction': direction, |
---|
| 173 | 'wait': wait, |
---|
| 174 | } |
---|
| 175 | code, value = self.emulab_call('experiment.swapexp', params) |
---|
| 176 | |
---|
| 177 | if self.log: |
---|
| 178 | if code == 0: self.log.info('[swap_exp]: Swap succeeded') |
---|
| 179 | else: self.log.error('[swap_exp]: Swap failed: %s' % value) |
---|
| 180 | |
---|
| 181 | return code == 0 |
---|
| 182 | |
---|
| 183 | def terminate_exp(self, pid, eid, wait=True): |
---|
| 184 | """ |
---|
| 185 | Completely terminate experiment |
---|
| 186 | """ |
---|
| 187 | if self.debug: |
---|
| 188 | if self.log: |
---|
| 189 | self.log.info("[swap_exp]: (debug) terminate %s" % eid) |
---|
| 190 | return True |
---|
| 191 | else: |
---|
| 192 | if self.log: |
---|
| 193 | self.log.info("[swap_exp]: Terminating %s" % eid) |
---|
| 194 | params = { |
---|
| 195 | 'proj': pid, |
---|
| 196 | 'exp': eid, |
---|
| 197 | 'wait': wait, |
---|
| 198 | } |
---|
| 199 | code, value = self.emulab_call('experiment.endexp', params) |
---|
| 200 | |
---|
| 201 | if self.log: |
---|
| 202 | if code == 0: self.log.info('[swap_exp]: Terminate succeeded') |
---|
| 203 | else: self.log.error('[swap_exp]: Terminate failed: %s' % value) |
---|
| 204 | |
---|
| 205 | return code == 0 |
---|
| 206 | |
---|
| 207 | def modify_exp(self, pid, eid, tcl, wait=True): |
---|
| 208 | if self.debug: |
---|
| 209 | self.log.info("[modify_exp]: (debug) Modifying %s" % eid) |
---|
| 210 | return True |
---|
| 211 | else: |
---|
| 212 | self.log.info("[modify_exp]: Modifying %s" % eid) |
---|
| 213 | params = { |
---|
| 214 | 'proj': pid, |
---|
| 215 | 'exp': eid, |
---|
| 216 | 'nsfilestr': tcl, |
---|
| 217 | 'wait': wait, |
---|
| 218 | 'reboot': True, |
---|
| 219 | 'restart_eventsys': True, |
---|
| 220 | } |
---|
| 221 | code, value = self.emulab_call('experiment.modify', params) |
---|
| 222 | if self.log: |
---|
| 223 | if code == 0: |
---|
| 224 | self.log.info('[modify_exp]: Modify succeeded') |
---|
| 225 | else: |
---|
| 226 | self.log.error('[modify_exp]: Modify failed: %s' \ |
---|
| 227 | % value) |
---|
| 228 | return code == 0 |
---|
| 229 | |
---|
| 230 | def get_osid_map(self): |
---|
| 231 | oses = { } |
---|
| 232 | code, osids = self.emulab_call('osid.getlist', {}) |
---|
| 233 | for key, val in osids.items(): |
---|
| 234 | val['imageid'] = key |
---|
| 235 | oses[val['osid']] = val |
---|
| 236 | return oses |
---|
| 237 | |
---|
| 238 | def get_mapping(self, pid, eid): |
---|
| 239 | """ |
---|
| 240 | Get the physical to virtual mapping from the expinfo command and save |
---|
| 241 | it in the self.map member. |
---|
| 242 | """ |
---|
| 243 | |
---|
| 244 | ev_active = ('ISUP', 'ALWAYSUP' ) |
---|
| 245 | ev_starting = ('REBOOTED', 'REBOOTING','PXEBOOTING', |
---|
| 246 | 'BOOTING', 'RELOADSETUP', 'RELOADING', 'RELOADDONE', |
---|
| 247 | 'RELOADDONEV2', 'TBSETUP') |
---|
| 248 | ev_terminating = ( 'SHUTDOWN' ) |
---|
| 249 | |
---|
| 250 | |
---|
| 251 | if self.debug: |
---|
| 252 | if self.log: |
---|
| 253 | self.log.info("[get_mapping] (debug) Generating mapping") |
---|
| 254 | return True |
---|
| 255 | else: |
---|
| 256 | if self.log: |
---|
| 257 | self.log.info("[get_mapping] Generating mapping") |
---|
| 258 | |
---|
| 259 | osidmap = self.get_osid_map() |
---|
| 260 | |
---|
| 261 | params = { |
---|
| 262 | 'proj': pid, |
---|
| 263 | 'exp': eid, |
---|
| 264 | 'aspect': 'mapping' |
---|
| 265 | } |
---|
| 266 | code, nodes = self.emulab_call('experiment.info', params) |
---|
| 267 | if code ==0: |
---|
| 268 | for k, v in nodes.items(): |
---|
| 269 | if v.get('erole', False) and 'pnode' in v: |
---|
| 270 | st = v.get('status', 'up') |
---|
| 271 | ev = v.get('eventstatus', 'ISUP') |
---|
| 272 | os = v.get('osid', None) |
---|
| 273 | |
---|
| 274 | if st == 'up': |
---|
| 275 | if ev in ev_active: st = 'active' |
---|
| 276 | elif ev in ev_starting: st = 'starting' |
---|
| 277 | elif ev in ev_terminating: st = 'terminating' |
---|
| 278 | else: st = 'failed' |
---|
| 279 | else: st = 'failed' |
---|
| 280 | |
---|
| 281 | if os and os in osidmap: |
---|
| 282 | osname = osidmap[os].get('OS', None) |
---|
| 283 | osversion = osidmap[os].get('version', None) |
---|
| 284 | osimage = "%s/%s" % \ |
---|
| 285 | ( osidmap[os].get('pid', ''), |
---|
| 286 | osidmap[os].get('imageid', '')) |
---|
| 287 | else: |
---|
| 288 | osname = osversion = osimage = None |
---|
| 289 | |
---|
| 290 | self.node[k] = self.node_info(k, v['pnode'], st, |
---|
| 291 | osname, osversion, osimage) |
---|
| 292 | if self.log: |
---|
| 293 | self.log.info("Mapping complete") |
---|
| 294 | return True |
---|
| 295 | else: |
---|
| 296 | raise service_error(service_error.internal, |
---|
| 297 | "Cannot get node mapping of segment:%s/%s" % (pid, eid)) |
---|
| 298 | def get_initial_image(self, node, top): |
---|
| 299 | for e in top.elements: |
---|
| 300 | if isinstance(e, topdl.Computer): |
---|
| 301 | if node == e.name: |
---|
| 302 | if e.os and len(e.os) == 1: |
---|
| 303 | return e.os[0].get_attribute( |
---|
| 304 | 'emulab_access:initial_image') |
---|
| 305 | return None |
---|
| 306 | |
---|
| 307 | |
---|
| 308 | def do_operation(self, op, lnode, pnode, params, top): |
---|
| 309 | """ |
---|
| 310 | Carry out operation on node in the given experiment. |
---|
| 311 | """ |
---|
| 312 | def get_param(params, name): |
---|
| 313 | if params is None: |
---|
| 314 | return None |
---|
| 315 | for d in params: |
---|
| 316 | if 'attribute' in d and d['attribute'] == name: |
---|
| 317 | return d.get('value', None) |
---|
| 318 | else: |
---|
| 319 | return None |
---|
| 320 | |
---|
| 321 | op = op.lower() |
---|
| 322 | |
---|
| 323 | if op == 'restore': |
---|
| 324 | state = get_param(params, 'state') |
---|
| 325 | if state is None: |
---|
| 326 | self.status.append(operation_status(lnode, |
---|
| 327 | operation_status.bad_param, 'No state to restore')) |
---|
| 328 | return False |
---|
| 329 | elif state == 'initial': |
---|
| 330 | image = self.get_initial_image(lnode, top) |
---|
| 331 | if image: |
---|
| 332 | pid, iid = image.split('/') |
---|
| 333 | p = {'nodes': pnode, 'imagename': iid, 'imageproj': pid, |
---|
| 334 | 'wait': False} |
---|
| 335 | code, result = self.emulab_call('node.reload', p) |
---|
| 336 | if code == 0: |
---|
| 337 | self.status.append(operation_status(lnode, |
---|
| 338 | operation_status.success, 'reloading')) |
---|
| 339 | return True |
---|
| 340 | else: |
---|
| 341 | self.status.append(operation_status(lnode, |
---|
| 342 | operation_status.federant, |
---|
| 343 | 'Error code: %d' % code)) |
---|
| 344 | return False |
---|
| 345 | else: |
---|
| 346 | self.status.append(operation_status(lnode, |
---|
| 347 | operation_status.federant, |
---|
| 348 | 'cannot find imageid??')) |
---|
| 349 | return False |
---|
| 350 | elif state == 'boot': |
---|
| 351 | p = {'nodes': pnode, 'wait': False} |
---|
| 352 | code, result = self.emulab_call('node.reboot', p) |
---|
| 353 | if code == 0: |
---|
| 354 | self.status.append(operation_status(lnode, |
---|
| 355 | operation_status.success, 'rebooting')) |
---|
| 356 | return True |
---|
| 357 | else: |
---|
| 358 | self.status.append(operation_status(lnode, |
---|
| 359 | operation_status.federant, 'Error code: %d' % code)) |
---|
| 360 | return False |
---|
| 361 | else: |
---|
| 362 | if '/' in state: |
---|
| 363 | pid, iid = state.split('/') |
---|
| 364 | else: |
---|
| 365 | pid = 'emulab-ops' |
---|
| 366 | iid = state |
---|
| 367 | |
---|
| 368 | p = {'nodes': pnode, 'imagename': iid, 'imageproj': pid, |
---|
| 369 | 'wait': False} |
---|
| 370 | code, result = self.emulab_call('node.reload', p) |
---|
| 371 | if code == 0: |
---|
| 372 | self.status.append(operation_status(lnode, |
---|
| 373 | operation_status.success, 'reloading')) |
---|
| 374 | return True |
---|
| 375 | else: |
---|
| 376 | self.status.append(operation_status(lnode, |
---|
| 377 | operation_status.federant, |
---|
| 378 | 'Error code: %d' % code)) |
---|
| 379 | return False |
---|
| 380 | else: |
---|
| 381 | self.status.append(operation_status(lnode, operation_status.unsupp)) |
---|
| 382 | return False |
---|
| 383 | |
---|