| 1 | #!/usr/local/bin/python |
|---|
| 2 | |
|---|
| 3 | import sys, os |
|---|
| 4 | import re |
|---|
| 5 | import os.path |
|---|
| 6 | |
|---|
| 7 | import tempfile |
|---|
| 8 | import subprocess |
|---|
| 9 | import logging |
|---|
| 10 | import time |
|---|
| 11 | import signal |
|---|
| 12 | |
|---|
| 13 | import util |
|---|
| 14 | |
|---|
| 15 | from deter import topdl |
|---|
| 16 | |
|---|
| 17 | from ssh_emulab_segment import ssh_emulab_segment |
|---|
| 18 | from xmlrpc_emulab_segment import xmlrpc_emulab_segment |
|---|
| 19 | |
|---|
| 20 | |
|---|
| 21 | class start_segment(ssh_emulab_segment, xmlrpc_emulab_segment): |
|---|
| 22 | def __init__(self, log=None, keyfile=None, debug=False, boss=None, |
|---|
| 23 | ops=None, cert=None): |
|---|
| 24 | ssh_emulab_segment.__init__(self, log=log, keyfile=keyfile, debug=debug) |
|---|
| 25 | xmlrpc_emulab_segment.__init__(self, boss=boss, ops=ops, cert=cert) |
|---|
| 26 | # XXX: better way to work this... |
|---|
| 27 | self.containerize = '/share/containers-dev/containerize.py --config /share/containers-dev/site.conf %s %s %s' |
|---|
| 28 | |
|---|
| 29 | def get_mapping(self, pid, eid): |
|---|
| 30 | """ |
|---|
| 31 | Containers's info reporting is not sufficient to support this yet |
|---|
| 32 | """ |
|---|
| 33 | pass |
|---|
| 34 | |
|---|
| 35 | def set_up_experiment_filespace(self, user, host, pid, eid, tmpdir): |
|---|
| 36 | """ |
|---|
| 37 | Send all the software and configuration files into the experiment's |
|---|
| 38 | file space. To reduce the number of ssh connections, we script many |
|---|
| 39 | changes and execute the script. |
|---|
| 40 | """ |
|---|
| 41 | # Configuration directories on the remote machine |
|---|
| 42 | proj_dir = "/proj/%s/exp/%s/tmp" % (pid, eid) |
|---|
| 43 | softdir = "/proj/%s/software/%s" % (pid, eid) |
|---|
| 44 | # Local software dir |
|---|
| 45 | lsoftdir = "%s/software" % tmpdir |
|---|
| 46 | |
|---|
| 47 | # Open up a temporary file to contain a script for setting up the |
|---|
| 48 | # filespace for the new experiment. |
|---|
| 49 | self.log.info("[start_segment]: creating script file") |
|---|
| 50 | try: |
|---|
| 51 | sf, scriptname = tempfile.mkstemp() |
|---|
| 52 | scriptfile = os.fdopen(sf, 'w') |
|---|
| 53 | except EnvironmentError: |
|---|
| 54 | return False |
|---|
| 55 | |
|---|
| 56 | scriptbase = os.path.basename(scriptname) |
|---|
| 57 | |
|---|
| 58 | # Script the filesystem changes |
|---|
| 59 | print >>scriptfile, "/bin/rm -rf %s" % proj_dir |
|---|
| 60 | # Clear and create the software directory |
|---|
| 61 | print >>scriptfile, "/bin/rm -rf %s/*" % softdir |
|---|
| 62 | print >>scriptfile, 'mkdir -p %s' % proj_dir |
|---|
| 63 | if os.path.isdir(lsoftdir): |
|---|
| 64 | print >>scriptfile, 'mkdir -p %s' % softdir |
|---|
| 65 | print >>scriptfile, "rm -f %s" % scriptbase |
|---|
| 66 | scriptfile.close() |
|---|
| 67 | |
|---|
| 68 | # Move the script to the remote machine |
|---|
| 69 | # XXX: could collide tempfile names on the remote host |
|---|
| 70 | if self.scp_file(scriptname, user, host, scriptbase): |
|---|
| 71 | os.remove(scriptname) |
|---|
| 72 | else: |
|---|
| 73 | return False |
|---|
| 74 | |
|---|
| 75 | # Execute the script (and the script's last line deletes it) |
|---|
| 76 | if not self.ssh_cmd(user, host, "sh -x %s" % scriptbase): |
|---|
| 77 | return False |
|---|
| 78 | |
|---|
| 79 | for f in os.listdir(tmpdir): |
|---|
| 80 | if not os.path.isdir("%s/%s" % (tmpdir, f)): |
|---|
| 81 | if not self.scp_file("%s/%s" % (tmpdir, f), user, host, |
|---|
| 82 | "%s/%s" % (proj_dir, f)): |
|---|
| 83 | return False |
|---|
| 84 | if os.path.isdir(lsoftdir): |
|---|
| 85 | for f in os.listdir(lsoftdir): |
|---|
| 86 | if not os.path.isdir("%s/%s" % (lsoftdir, f)): |
|---|
| 87 | if not self.scp_file("%s/%s" % (lsoftdir, f), |
|---|
| 88 | user, host, "%s/%s" % (softdir, f)): |
|---|
| 89 | return False |
|---|
| 90 | return True |
|---|
| 91 | |
|---|
| 92 | |
|---|
| 93 | def __call__(self, parent, eid, pid, user, tclfile, tmpdir, timeout=0, |
|---|
| 94 | gid=None): |
|---|
| 95 | """ |
|---|
| 96 | Start a sub-experiment on a federant. |
|---|
| 97 | |
|---|
| 98 | Get the current state, and terminate the experiment if it exists. The |
|---|
| 99 | group membership of the experiment is difficult to determine or change, |
|---|
| 100 | so start with a clean slate. Create a new one and ship data |
|---|
| 101 | and configs and start the experiment. There are small ordering |
|---|
| 102 | differences based on the initial state of the sub-experiment. |
|---|
| 103 | """ |
|---|
| 104 | |
|---|
| 105 | # containers listens to the filename extension. tclfile here is really |
|---|
| 106 | # topdl, but we'll move it to a different filename. |
|---|
| 107 | if tclfile.endswith('.tcl'): |
|---|
| 108 | topdlfile = os.path.join(tmpdir, 'experiment.xml') |
|---|
| 109 | self.ssh_cmd(user, self.ops, 'mv %s %s' % (tclfile, topdlfile)) |
|---|
| 110 | else: |
|---|
| 111 | topdlfile = tclfile |
|---|
| 112 | |
|---|
| 113 | |
|---|
| 114 | state = self.get_state(pid, eid) |
|---|
| 115 | |
|---|
| 116 | if state != 'none': |
|---|
| 117 | self.terminate_exp(pid, eid) |
|---|
| 118 | |
|---|
| 119 | cmd = self.containerize % (pid, eid, topdlfile) |
|---|
| 120 | |
|---|
| 121 | if not self.ssh_cmd(user, self.ops, cmd): |
|---|
| 122 | self.log.error("Cannot create containers experiment") |
|---|
| 123 | return False |
|---|
| 124 | |
|---|
| 125 | if not self.set_up_experiment_filespace(user, self.ops, |
|---|
| 126 | pid, eid, tmpdir): |
|---|
| 127 | return False |
|---|
| 128 | |
|---|
| 129 | if not self.swap_exp(pid, eid, 'in'): |
|---|
| 130 | self.log.error("swap in failed") |
|---|
| 131 | return False |
|---|
| 132 | # Everything has gone OK. |
|---|
| 133 | self.get_mapping(pid,eid) |
|---|
| 134 | return True |
|---|
| 135 | |
|---|
| 136 | class stop_segment(ssh_emulab_segment, xmlrpc_emulab_segment): |
|---|
| 137 | def __init__(self, log=None, keyfile=None, debug=False, boss=None, |
|---|
| 138 | ops=None, cert=None): |
|---|
| 139 | ssh_emulab_segment.__init__(self, log=log, keyfile=keyfile, debug=debug) |
|---|
| 140 | xmlrpc_emulab_segment.__init__(self, boss=boss, ops=ops, cert=cert) |
|---|
| 141 | |
|---|
| 142 | def __call__(self, parent, user, pid, eid, gid=None, terminate=False): |
|---|
| 143 | """ |
|---|
| 144 | Stop a sub experiment by calling swapexp on the federant |
|---|
| 145 | """ |
|---|
| 146 | |
|---|
| 147 | self.log.info("[stop_segment]: Stopping %s" % eid) |
|---|
| 148 | rv = False |
|---|
| 149 | try: |
|---|
| 150 | # Clean out tar files: we've gone over quota in the past |
|---|
| 151 | self.ssh_cmd(user, self.ops, |
|---|
| 152 | "rm -rf /proj/%s/software/%s" % (pid, eid)) |
|---|
| 153 | rv = self.swap_exp(pid, eid, 'out') |
|---|
| 154 | if terminate: |
|---|
| 155 | rv = self.terminate_exp(pid, eid) |
|---|
| 156 | except self.cmd_timeout: |
|---|
| 157 | rv = False |
|---|
| 158 | return rv |
|---|
| 159 | |
|---|
| 160 | class info_segment(ssh_emulab_segment, xmlrpc_emulab_segment): |
|---|
| 161 | def __init__(self, log=None, keyfile=None, debug=False, boss=None, |
|---|
| 162 | ops=None, cert=None): |
|---|
| 163 | ssh_emulab_segment.__init__(self, log=log, keyfile=keyfile, debug=debug) |
|---|
| 164 | xmlrpc_emulab_segment.__init__(self, boss=boss, ops=ops, cert=cert) |
|---|
| 165 | |
|---|
| 166 | def get_mapping(self, pid, eid): |
|---|
| 167 | """ |
|---|
| 168 | Benito's info reporting is not sufficient to support this yet |
|---|
| 169 | """ |
|---|
| 170 | pass |
|---|
| 171 | |
|---|
| 172 | def __call__(self, parent, user, pid, eid): |
|---|
| 173 | self.log.info("[info_segment]: Getting info from %s" % eid) |
|---|
| 174 | self.get_mapping(pid,eid) |
|---|
| 175 | return True |
|---|
| 176 | |
|---|
| 177 | class operation_segment(ssh_emulab_segment, xmlrpc_emulab_segment): |
|---|
| 178 | def __init__(self, log=None, keyfile=None, debug=False, boss=None, |
|---|
| 179 | ops=None, cert=None): |
|---|
| 180 | ssh_emulab_segment.__init__(self, log=log, keyfile=keyfile, debug=debug) |
|---|
| 181 | xmlrpc_emulab_segment.__init__(self, boss=boss, ops=ops, cert=cert) |
|---|
| 182 | |
|---|
| 183 | def __call__(self, parent, op, targets, param, top): |
|---|
| 184 | self.status.append('operations not yet supported') |
|---|
| 185 | return True |
|---|
