[22c88c9] | 1 | #!/usr/local/bin/python |
---|
| 2 | |
---|
| 3 | import sys |
---|
| 4 | import re |
---|
| 5 | import os |
---|
[b14b495] | 6 | import random |
---|
[22c88c9] | 7 | |
---|
[ec9962b] | 8 | from optparse import OptionParser, OptionValueError |
---|
[22c88c9] | 9 | from federation.remote_service import service_caller |
---|
| 10 | from federation import topdl |
---|
| 11 | |
---|
| 12 | class constraint: |
---|
[b0581ac] | 13 | def __init__(self, name=None, required=False, accepts=None, provides=None, |
---|
| 14 | match=None): |
---|
| 15 | self.name = name |
---|
[22c88c9] | 16 | self.required = required |
---|
| 17 | self.accepts = accepts or [] |
---|
| 18 | self.provides = provides or [] |
---|
[b0581ac] | 19 | self.match = match |
---|
[22c88c9] | 20 | |
---|
| 21 | def __str__(self): |
---|
[b0581ac] | 22 | return "%s:%s:%s:%s" % (self.name, self.required, |
---|
| 23 | ",".join(self.provides), ",".join(self.accepts)) |
---|
[22c88c9] | 24 | |
---|
| 25 | |
---|
[b0581ac] | 26 | def add_to_map(exp, mark, provides, accepts): |
---|
| 27 | """ |
---|
| 28 | Create a constraint from a string of the form |
---|
| 29 | name:required:provides:accepts |
---|
| 30 | where name is the name of the node in the current topology, if the second |
---|
| 31 | field is "required" this is a required constraint, a list of attributes |
---|
| 32 | that this connection point provides and a list that it accepts. The |
---|
| 33 | provides and accepts lists are comma-separated. Note that the string |
---|
| 34 | passed in as exp should have whitespace removed. The constraint is added to |
---|
| 35 | the mark dict under the name key and entries in the provides and accepts |
---|
| 36 | are made to teh constraing for each attribute that it provides or accepts. |
---|
| 37 | """ |
---|
[22c88c9] | 38 | nn, r, p, a = exp.split(":") |
---|
| 39 | p = p.split(",") |
---|
| 40 | a = a.split(",") |
---|
[b0581ac] | 41 | c = constraint(name=nn, required=(r == 'required'), provides=p, accepts=a) |
---|
| 42 | mark[nn] = c |
---|
[22c88c9] | 43 | |
---|
[b0581ac] | 44 | for attr, dict in ((p, provides), (a, accepts)): |
---|
| 45 | for a in attr: |
---|
| 46 | if a not in dict: dict[a] = [c] |
---|
| 47 | else: dict[a].append(c) |
---|
[22c88c9] | 48 | |
---|
[b0581ac] | 49 | def make_new_name(names, prefix="name"): |
---|
| 50 | """ |
---|
| 51 | Generate an identifier not present in names by appending an integer to |
---|
| 52 | prefix. The new name is inserted into names and returned. |
---|
| 53 | """ |
---|
| 54 | i = 0 |
---|
| 55 | n = "%s%d" % (prefix,i) |
---|
| 56 | while n in names: |
---|
| 57 | i += 1 |
---|
| 58 | n = "%s%d" % (prefix,i) |
---|
| 59 | names.add(n) |
---|
| 60 | return n |
---|
[22c88c9] | 61 | |
---|
[b0581ac] | 62 | def add_interfaces(top, mark): |
---|
| 63 | """ |
---|
| 64 | Add unconnected interfaces to the nodes whose names are keys in the mark |
---|
| 65 | dict. As the interface is added change the name field of the contstraint |
---|
| 66 | in mark into a tuple containing the node name, interface name, and topology |
---|
| 67 | in which they reside. |
---|
| 68 | """ |
---|
[22c88c9] | 69 | for e in top.elements: |
---|
[b0581ac] | 70 | if e.name in mark: |
---|
| 71 | con = mark[e.name] |
---|
| 72 | ii = make_new_name(set([x.name for x in e.interface]), "inf") |
---|
| 73 | e.interface.append( |
---|
| 74 | topdl.Interface(substrate=[], name=ii, element=e)) |
---|
| 75 | con.name = (e.name, ii, top) |
---|
[22c88c9] | 76 | |
---|
[b0581ac] | 77 | def localize_names(top, names, marks): |
---|
| 78 | """ |
---|
| 79 | Take a topology and rename any substrates or elements that share a name |
---|
| 80 | with an existing computer or substrate. Keep the original name as a |
---|
| 81 | localized_name attribute. In addition, remap any constraints or interfaces |
---|
| 82 | that point to the old name over to the new one. Constraints are found in |
---|
| 83 | the marks dict, indexed by node name. Those constraints name attributes |
---|
| 84 | have already been converted to triples (node name, interface name, |
---|
| 85 | topology) so only the node name needs to be changed. |
---|
| 86 | """ |
---|
| 87 | sub_map = { } |
---|
| 88 | for s in top.substrates: |
---|
| 89 | s.set_attribute('localized_name', s.name) |
---|
| 90 | if s.name in names: |
---|
| 91 | sub_map[s.name] = n = make_new_name(names, "substrate") |
---|
| 92 | s.name = n |
---|
| 93 | else: |
---|
| 94 | names.add(s.name) |
---|
| 95 | |
---|
| 96 | for e in [ e for e in top.elements if isinstance(e, topdl.Computer)]: |
---|
| 97 | e.set_attribute('localized_name', e.name) |
---|
| 98 | if e.name in names: |
---|
| 99 | nn= make_new_name(names, "computer") |
---|
| 100 | if e.name in marks: |
---|
| 101 | n, i, t = marks[e.name].name |
---|
| 102 | marks[e.name].name = (nn, i, t) |
---|
| 103 | e.name = nn |
---|
| 104 | else: |
---|
| 105 | names.add(e.name) |
---|
| 106 | |
---|
| 107 | # Interface mapping. The list comprehension creates a list of |
---|
| 108 | # substrate names where each element in the list is replaced by the |
---|
| 109 | # entry in sub_map indexed by it if present and left alone otherwise. |
---|
| 110 | for i in e.interface: |
---|
| 111 | i.substrate = [ sub_map.get(ii, ii) for ii in i.substrate ] |
---|
| 112 | |
---|
| 113 | def meet_constraints(candidates, provides, accepts): |
---|
| 114 | """ |
---|
| 115 | Try to meet all the constraints in candidates using the information in the |
---|
| 116 | provides and accepts dicts (which index constraints that provide or accept |
---|
| 117 | the given attribute). A constraint is met if it can be matched with another |
---|
| 118 | constraint that provides an attribute that the first constraint accepts. |
---|
| 119 | Only one match per pair is allowed, and we always prefer matching a |
---|
| 120 | required constraint to an unreqired one. If all the candidates can be |
---|
| 121 | matches, return True and return False otherwise. |
---|
| 122 | """ |
---|
| 123 | got_all = True |
---|
| 124 | for c in candidates: |
---|
| 125 | if not c.match: |
---|
| 126 | match = None |
---|
| 127 | for a in c.accepts: |
---|
| 128 | for can in provides.get(a,[]): |
---|
| 129 | # A constraint cannot satisfy itself, nor do we allow loops |
---|
| 130 | # within the same topology. This also excludes matched |
---|
| 131 | # constraints. |
---|
| 132 | if can.name != c.name and can.name[2] != c.name[2] and \ |
---|
| 133 | not can.match: |
---|
| 134 | match = can |
---|
| 135 | # Within providers, prefer matches against required |
---|
| 136 | # composition points |
---|
| 137 | if can.required: |
---|
| 138 | break |
---|
| 139 | # Within acceptance categories, prefer matches against required |
---|
| 140 | # composition points |
---|
| 141 | if match and match.required: |
---|
| 142 | break |
---|
| 143 | |
---|
| 144 | # Done checking all possible matches. |
---|
| 145 | if match: |
---|
| 146 | match.match = c |
---|
| 147 | c.match = match |
---|
| 148 | else: |
---|
| 149 | got_all = False |
---|
| 150 | return got_all |
---|
[22c88c9] | 151 | |
---|
[b14b495] | 152 | def randomize_constraint_order(indexes): |
---|
| 153 | """ |
---|
| 154 | Randomly reorder the lists of constraints that provides and accepts hold. |
---|
| 155 | """ |
---|
| 156 | if not isinstance(indexes, tuple): |
---|
| 157 | indexes = (indexes,) |
---|
| 158 | |
---|
| 159 | for idx in indexes: |
---|
| 160 | for k in idx.keys(): |
---|
| 161 | random.shuffle(idx[k]) |
---|
| 162 | |
---|
[22c88c9] | 163 | def remote_ns2topdl(uri, desc, cert): |
---|
[b0581ac] | 164 | """ |
---|
| 165 | Call a remote service to convert the ns2 to topdl (and in fact all the way |
---|
| 166 | to a topdl.Topology. |
---|
| 167 | """ |
---|
[22c88c9] | 168 | |
---|
| 169 | req = { 'description' : { 'ns2description': desc }, } |
---|
| 170 | |
---|
| 171 | try: |
---|
| 172 | r = service_caller('Ns2Topdl')(uri, req, cert) |
---|
| 173 | except: |
---|
| 174 | return None |
---|
| 175 | |
---|
| 176 | if r.has_key('Ns2TopdlResponseBody'): |
---|
| 177 | r = r['Ns2TopdlResponseBody'] |
---|
| 178 | ed = r.get('experimentdescription', None) |
---|
| 179 | if 'topdldescription' in ed: |
---|
| 180 | return topdl.Topology(**ed['topdldescription']) |
---|
| 181 | else: |
---|
| 182 | return None |
---|
| 183 | else: |
---|
| 184 | return None |
---|
| 185 | |
---|
[b0581ac] | 186 | def connect_composition_points(top, contraints): |
---|
| 187 | """ |
---|
| 188 | top is a topology containing copies of all the topologies represented in |
---|
| 189 | the contsraints, flattened into one name space. This routine inserts the |
---|
| 190 | additional substrates required to interconnect the topology as described by |
---|
| 191 | the constraints. After the links are made, unused connection points are |
---|
| 192 | purged. |
---|
| 193 | """ |
---|
| 194 | done = set() |
---|
| 195 | for c in constraints: |
---|
| 196 | if c not in done and c.match: |
---|
| 197 | # c is an unprocessed matched constraint. Create a substrate |
---|
| 198 | # and attach it to the interfaces named by c and c.match. |
---|
| 199 | sn = make_new_name(names, "sub") |
---|
| 200 | s = topdl.Substrate(name=sn) |
---|
| 201 | connected = 0 |
---|
| 202 | # Walk through all the computers in the topology |
---|
| 203 | for e in [ e for e in top.elements \ |
---|
| 204 | if isinstance(e, topdl.Computer)]: |
---|
| 205 | # if the computer matches the computer and interface name in |
---|
| 206 | # either c or c.match, connect it. Once both computers have |
---|
| 207 | # been found and connected, exit the loop walking through all |
---|
| 208 | # computers. |
---|
| 209 | for comp, inf in (c.name[0:2], c.match.name[0:2]): |
---|
| 210 | if e.name == comp: |
---|
| 211 | for i in e.interface: |
---|
| 212 | if i.name == inf: |
---|
| 213 | i.substrate.append(sn) |
---|
| 214 | connected += 1 |
---|
| 215 | break |
---|
| 216 | break |
---|
| 217 | # Connected both, so add the substrate to the topology |
---|
| 218 | if connected == 2: |
---|
| 219 | top.substrates.append(s) |
---|
| 220 | break |
---|
| 221 | # c and c.match have been processed, so add them to the done set |
---|
| 222 | done.add(c) |
---|
| 223 | done.add(c.match) |
---|
| 224 | |
---|
| 225 | # All interfaces with matched constraints have been connected. Cull any |
---|
| 226 | # interfaces unassigned to a substrate |
---|
| 227 | for e in [ e for e in top.elements if isinstance(e, topdl.Computer)]: |
---|
| 228 | if any([ not i.substrate for i in e.interface]): |
---|
| 229 | e.interface = [ i for i in e.interface if i.substrate ] |
---|
| 230 | |
---|
| 231 | top.incorporate_elements() |
---|
| 232 | |
---|
[ec9962b] | 233 | def import_tcl_constraints(marks, provides, accepts, contents): |
---|
| 234 | """ |
---|
| 235 | Contents is a list containing the lines of an annotated ns2 file. This |
---|
| 236 | routine extracts the constraint comment lines and convertes them into |
---|
| 237 | constraints in the namespace of the tcl experiment, as well as inserting |
---|
| 238 | them in the accepts and provides indices. |
---|
| 239 | """ |
---|
| 240 | for l in contents: |
---|
| 241 | m = const_re.search(l) |
---|
| 242 | if m: |
---|
| 243 | add_to_map(re.sub('\s', '', m.group(1)), marks, provides, |
---|
| 244 | accepts) |
---|
| 245 | |
---|
| 246 | |
---|
| 247 | def multi_callback(option, opt_str, value, parser): |
---|
| 248 | """ |
---|
| 249 | Parse a --multifile command line option. The parameter is of the form |
---|
| 250 | filename,count. This splits the argument at the rightmost comma and |
---|
| 251 | inserts the filename, count tuple into the "files" option. It handles a |
---|
| 252 | couple error cases, too. This is an optparse.OptionParser callback. |
---|
| 253 | """ |
---|
| 254 | idx = value.rfind(',') |
---|
| 255 | if idx != -1: |
---|
| 256 | try: |
---|
| 257 | parser.values.files.append((value[0:idx], int(value[idx+1:]))) |
---|
| 258 | except ValueError, e: |
---|
| 259 | raise OptionValueError("Can't convert %s to int in multifile (%s)" \ |
---|
| 260 | % (value[idx+1:], value)) |
---|
| 261 | else: |
---|
| 262 | raise OptionValueError("Bad format (need a comma) for multifile: %s" \ |
---|
| 263 | % value) |
---|
| 264 | |
---|
| 265 | |
---|
[b0581ac] | 266 | |
---|
[da5b93c] | 267 | # Main line begins |
---|
[b0581ac] | 268 | |
---|
[22c88c9] | 269 | const_re = re.compile("\s*#\s*COMPOSITION:\s*([^:]+:[^:]+:.*)") |
---|
| 270 | |
---|
| 271 | parser = OptionParser() |
---|
| 272 | parser.add_option('--url', dest='url', default="http://localhost:13235", |
---|
| 273 | help='url of ns2 to topdl service') |
---|
| 274 | parser.add_option('--certfile', dest='cert', default=None, |
---|
| 275 | help='Certificate to use as identity') |
---|
[b14b495] | 276 | parser.add_option('--seed', dest='seed', type='int', default=None, |
---|
| 277 | help='Random number seed') |
---|
[ec9962b] | 278 | parser.add_option('--multifile', dest='files', default=[], type='string', |
---|
| 279 | action='callback', callback=multi_callback, |
---|
| 280 | help="Include file multiple times") |
---|
[22c88c9] | 281 | |
---|
| 282 | opts, args = parser.parse_args() |
---|
| 283 | |
---|
| 284 | if opts.cert: |
---|
| 285 | cert = opts.cert |
---|
| 286 | elif os.access(os.path.expanduser("~/.ssl/emulab.pem"), os.R_OK): |
---|
| 287 | cert = os.path.expanduser("~/.ssl/emulab.pem") |
---|
| 288 | else: |
---|
| 289 | cert = None |
---|
| 290 | |
---|
[b14b495] | 291 | random.seed(opts.seed) |
---|
| 292 | |
---|
[ec9962b] | 293 | files = opts.files |
---|
| 294 | files.extend([ (a, 1) for a in args]) |
---|
| 295 | |
---|
[b0581ac] | 296 | comps = [ ] |
---|
| 297 | names = set() |
---|
| 298 | constraints = [ ] |
---|
| 299 | provides = { } |
---|
| 300 | accepts = { } |
---|
[ec9962b] | 301 | for fn, cnt in files: |
---|
[22c88c9] | 302 | marks = { } |
---|
| 303 | try: |
---|
| 304 | f = open(fn, "r") |
---|
[ec9962b] | 305 | contents = [ l for l in f ] |
---|
[22c88c9] | 306 | except EnvironmentError, e: |
---|
[b0581ac] | 307 | print >>sys.stderr, "Error on %s: %s" % (fn, e) |
---|
| 308 | continue |
---|
[22c88c9] | 309 | |
---|
[ec9962b] | 310 | top = remote_ns2topdl(opts.url, "".join(contents), cert) |
---|
[da5b93c] | 311 | if not top: |
---|
| 312 | sys.exit("Cannot create topology from: %s" % fn) |
---|
[ec9962b] | 313 | |
---|
| 314 | for i in range(0, cnt): |
---|
| 315 | t = top.clone() |
---|
| 316 | import_tcl_constraints(marks, provides, accepts, contents) |
---|
| 317 | add_interfaces(t, marks) |
---|
| 318 | localize_names(t, names, marks) |
---|
| 319 | t.incorporate_elements() |
---|
| 320 | constraints.extend(marks.values()) |
---|
| 321 | comps.append(t) |
---|
[22c88c9] | 322 | |
---|
[da5b93c] | 323 | # Let the user know if they messed up on specifying constraints. |
---|
| 324 | if any([ not isinstance(c.name, tuple) for c in constraints]): |
---|
| 325 | print >>sys.stderr, "nodes not found for constraints on %s" % \ |
---|
| 326 | ",".join([ c.name for c in constraints \ |
---|
| 327 | if isinstance(c.name, basestring)]) |
---|
| 328 | constraints = [ c for c in constraints if isinstance(c.name, tuple )] |
---|
| 329 | |
---|
[b14b495] | 330 | # Mix up the constraint indexes |
---|
| 331 | randomize_constraint_order((provides, accepts)) |
---|
| 332 | |
---|
[b0581ac] | 333 | # Now the various components live in the same namespace and are marked with |
---|
| 334 | # their composition requirements. |
---|
| 335 | |
---|
| 336 | if not meet_constraints([c for c in constraints if c.required], |
---|
| 337 | provides, accepts): |
---|
| 338 | sys.exit("Could not meet all required constraints") |
---|
| 339 | meet_constraints([ c for c in constraints if not c.match ], provides, accepts) |
---|
| 340 | |
---|
| 341 | # Make a topology containing all elements and substrates from components that |
---|
| 342 | # had matches. |
---|
| 343 | comp = topdl.Topology() |
---|
| 344 | for t in set([ c.name[2] for c in constraints if c.match]): |
---|
| 345 | comp.elements.extend([e.clone() for e in t.elements]) |
---|
| 346 | comp.substrates.extend([s.clone() for s in t.substrates]) |
---|
| 347 | |
---|
| 348 | # Add substrates and connections corresponding to composition points |
---|
| 349 | connect_composition_points(comp, constraints) |
---|
| 350 | |
---|
| 351 | print topdl.topology_to_xml(comp, top='experiment') |
---|