[22c88c9] | 1 | #!/usr/local/bin/python |
---|
| 2 | |
---|
| 3 | import sys |
---|
| 4 | import re |
---|
| 5 | import os |
---|
[b14b495] | 6 | import random |
---|
[5f6c3af] | 7 | import copy |
---|
[22c88c9] | 8 | |
---|
[4d4dde4] | 9 | import xml.parsers.expat |
---|
| 10 | |
---|
[ec9962b] | 11 | from optparse import OptionParser, OptionValueError |
---|
[22c88c9] | 12 | from federation.remote_service import service_caller |
---|
[5f6c3af] | 13 | from federation.service_error import service_error |
---|
[22c88c9] | 14 | from federation import topdl |
---|
| 15 | |
---|
[9e38ded] | 16 | class config_error(RuntimeError): pass |
---|
| 17 | |
---|
[22c88c9] | 18 | class constraint: |
---|
[4b6909e] | 19 | """ |
---|
[8a0c67f] | 20 | This is mainly a struct to hold constraint fields and convert to XML output |
---|
[4b6909e] | 21 | """ |
---|
[b0581ac] | 22 | def __init__(self, name=None, required=False, accepts=None, provides=None, |
---|
[89e2138] | 23 | topology=None, match=None): |
---|
[b0581ac] | 24 | self.name = name |
---|
[22c88c9] | 25 | self.required = required |
---|
| 26 | self.accepts = accepts or [] |
---|
| 27 | self.provides = provides or [] |
---|
[89e2138] | 28 | self.topology = None |
---|
[b0581ac] | 29 | self.match = match |
---|
[22c88c9] | 30 | |
---|
| 31 | def __str__(self): |
---|
[b0581ac] | 32 | return "%s:%s:%s:%s" % (self.name, self.required, |
---|
| 33 | ",".join(self.provides), ",".join(self.accepts)) |
---|
[22c88c9] | 34 | |
---|
[4d4dde4] | 35 | def to_xml(self): |
---|
| 36 | rv = "<constraint>" |
---|
[89e2138] | 37 | rv += "<name>%s</name>" % self.name |
---|
[4d4dde4] | 38 | rv += "<required>%s</required>" % self.required |
---|
| 39 | for p in self.provides: |
---|
| 40 | rv += "<provides>%s</provides>" % p |
---|
| 41 | for a in self.accepts: |
---|
| 42 | rv += "<accepts>%s</accepts>" % a |
---|
| 43 | rv+= "</constraint>" |
---|
| 44 | return rv |
---|
| 45 | |
---|
| 46 | def constraints_from_xml(string=None, file=None, filename=None, |
---|
| 47 | top="constraints"): |
---|
[8a0c67f] | 48 | """ |
---|
| 49 | Pull constraints from an xml file. Only constraints in the top element are |
---|
| 50 | recognized. A constraint consists of a constraint element with one name, |
---|
| 51 | one required, and multiple accepts nd provides elements. Each contains a |
---|
| 52 | string. A list of constraints is returned. |
---|
| 53 | """ |
---|
[4d4dde4] | 54 | class parser: |
---|
[8a0c67f] | 55 | """ |
---|
| 56 | A little class to encapsulate some state used to parse the constraints. |
---|
| 57 | The methods are all bound to the analogous handlers in an XML parser. |
---|
| 58 | It collects the constraints in self.constraint. |
---|
| 59 | """ |
---|
[4d4dde4] | 60 | def __init__(self, top): |
---|
| 61 | self.top = top |
---|
| 62 | self.constraints = [ ] |
---|
| 63 | self.chars = None |
---|
| 64 | self.current = None |
---|
| 65 | self.in_top = False |
---|
[5e4025d] | 66 | |
---|
[4d4dde4] | 67 | def start_element(self, name, attrs): |
---|
[8a0c67f] | 68 | # Clear any collected strings (from inside elements) |
---|
[4d4dde4] | 69 | self.chars = None |
---|
| 70 | self.key = str(name) |
---|
[8a0c67f] | 71 | |
---|
| 72 | # See if we've entered the containing context |
---|
[4d4dde4] | 73 | if name == self.top: |
---|
| 74 | self.in_top = True |
---|
| 75 | |
---|
[8a0c67f] | 76 | # Entering a constraint, create the object which also acts as a |
---|
| 77 | # flag to indicate we're collecting constraint data. |
---|
[4d4dde4] | 78 | if self.in_top: |
---|
| 79 | if name == 'constraint': |
---|
| 80 | self.current = constraint() |
---|
| 81 | |
---|
| 82 | def end_element(self, name): |
---|
| 83 | if self.current: |
---|
[8a0c67f] | 84 | # In a constraint and leaving an element. Clean up any string |
---|
| 85 | # we've collected and process elements we know. |
---|
[4d4dde4] | 86 | if self.chars is not None: |
---|
| 87 | self.chars = self.chars.strip() |
---|
| 88 | |
---|
| 89 | if name == 'required': |
---|
| 90 | if self.chars is None: |
---|
| 91 | self.current.required = False |
---|
| 92 | else: |
---|
| 93 | self.current.required = (self.chars.lower() == 'true') |
---|
| 94 | elif name == 'name': |
---|
| 95 | self.current.name = self.chars |
---|
| 96 | elif name == 'accepts': |
---|
| 97 | self.current.accepts.append(self.chars) |
---|
| 98 | elif name == 'provides': |
---|
| 99 | self.current.provides.append(self.chars) |
---|
| 100 | elif name == 'constraint': |
---|
[8a0c67f] | 101 | # Leaving this constraint. Record it and clear the flag |
---|
[4d4dde4] | 102 | self.constraints.append(self.current) |
---|
| 103 | self.current = None |
---|
| 104 | else: |
---|
| 105 | print >>sys.stderr, \ |
---|
| 106 | "Unexpected element in constraint: %s" % name |
---|
| 107 | elif name == self.top: |
---|
[8a0c67f] | 108 | # We've left the containing context |
---|
[4d4dde4] | 109 | self.in_top = False |
---|
| 110 | |
---|
| 111 | |
---|
| 112 | def char_data(self, data): |
---|
[8a0c67f] | 113 | # Collect strings if we're in the overall context |
---|
[4d4dde4] | 114 | if self.in_top: |
---|
| 115 | if self.chars is None: self.chars = data |
---|
| 116 | else: self.chars += data |
---|
| 117 | |
---|
[8a0c67f] | 118 | # Beginning of constraints_from_xml. Connect up the parser and call it |
---|
| 119 | # properly for the kind of input supplied. |
---|
[4d4dde4] | 120 | p = parser(top=top) |
---|
| 121 | xp = xml.parsers.expat.ParserCreate() |
---|
| 122 | |
---|
| 123 | xp.StartElementHandler = p.start_element |
---|
| 124 | xp.EndElementHandler = p.end_element |
---|
| 125 | xp.CharacterDataHandler = p.char_data |
---|
| 126 | |
---|
| 127 | if len([x for x in (string, filename, file) if x is not None])!= 1: |
---|
| 128 | raise RuntimeError("Exactly one one of file, filename and string " + \ |
---|
| 129 | "must be set") |
---|
| 130 | elif filename: |
---|
| 131 | f = open(filename, "r") |
---|
| 132 | xp.ParseFile(f) |
---|
| 133 | f.close() |
---|
| 134 | elif file: |
---|
| 135 | xp.ParseFile(file) |
---|
| 136 | elif string: |
---|
| 137 | xp.Parse(string, isfinal=True) |
---|
| 138 | else: |
---|
| 139 | return [] |
---|
| 140 | |
---|
| 141 | return p.constraints |
---|
| 142 | |
---|
| 143 | |
---|
[5e4025d] | 144 | class ComposeOptionParser(OptionParser): |
---|
| 145 | """ |
---|
| 146 | This class encapsulates the options to this script in one place. It also |
---|
| 147 | holds the callback for the multifile choice. |
---|
| 148 | """ |
---|
| 149 | def __init__(self): |
---|
| 150 | OptionParser.__init__(self) |
---|
| 151 | self.add_option('--url', dest='url', default="http://localhost:13235", |
---|
| 152 | help='url of ns2 to topdl service') |
---|
| 153 | self.add_option('--certfile', dest='cert', default=None, |
---|
| 154 | help='Certificate to use as identity') |
---|
| 155 | self.add_option('--seed', dest='seed', type='int', default=None, |
---|
| 156 | help='Random number seed') |
---|
| 157 | self.add_option('--multifile', dest='files', default=[], type='string', |
---|
| 158 | action='callback', callback=self.multi_callback, |
---|
| 159 | help="Include file multiple times") |
---|
| 160 | self.add_option('--output', dest='outfile', default=None, |
---|
| 161 | help='Output file name') |
---|
[4d4dde4] | 162 | self.add_option("--format", dest="format", type="choice", |
---|
| 163 | choices=("xml", "topdl", "tcl", "ns"), |
---|
| 164 | help="Output file format") |
---|
[023e79b] | 165 | self.add_option('--add_testbeds', dest='add_testbeds', default=False, |
---|
| 166 | action='store_true', |
---|
| 167 | help='add testbed attributes to each component') |
---|
[2c51061] | 168 | self.add_option('--output_testbeds', dest='output_testbeds', |
---|
| 169 | default=False, action='store_true', |
---|
| 170 | help='Output tb-set-node-testbed commands to ns2') |
---|
[6df4b11] | 171 | self.add_option('--lax', dest='lax', default=False, |
---|
| 172 | action='store_true', |
---|
| 173 | help='allow solutions where unrequired constraints fail') |
---|
[966c620] | 174 | self.add_option('--same_node', dest='same_node', action='store_true', |
---|
| 175 | default=False, |
---|
| 176 | help='Allow loops to the same node to be created.') |
---|
| 177 | self.add_option('--same_topology', dest='same_topo', |
---|
| 178 | action='store_true', default=False, |
---|
| 179 | help='Allow links within the same topology to be created.') |
---|
| 180 | self.add_option('--same_pair', dest='multi_pair', |
---|
| 181 | action='store_true', default=False, |
---|
| 182 | help='Allow multiple links between the same nodes " + \ |
---|
| 183 | "to be created.') |
---|
[9e38ded] | 184 | self.add_option('--config', dest='config', default=None, |
---|
| 185 | help='Configuration file of options') |
---|
[5e4025d] | 186 | |
---|
| 187 | @staticmethod |
---|
| 188 | def multi_callback(option, opt_str, value, parser): |
---|
| 189 | """ |
---|
| 190 | Parse a --multifile command line option. The parameter is of the form |
---|
| 191 | filename,count. This splits the argument at the rightmost comma and |
---|
| 192 | inserts the filename, count tuple into the "files" option. It handles |
---|
| 193 | a couple error cases, too. |
---|
| 194 | """ |
---|
| 195 | idx = value.rfind(',') |
---|
| 196 | if idx != -1: |
---|
| 197 | try: |
---|
| 198 | parser.values.files.append((value[0:idx], int(value[idx+1:]))) |
---|
| 199 | except ValueError, e: |
---|
| 200 | raise OptionValueError( |
---|
| 201 | "Can't convert %s to int in multifile (%s)" % \ |
---|
| 202 | (value[idx+1:], value)) |
---|
| 203 | else: |
---|
| 204 | raise OptionValueError( |
---|
| 205 | "Bad format (need a comma) for multifile: %s" % value) |
---|
| 206 | |
---|
| 207 | def warn(msg): |
---|
| 208 | """ |
---|
| 209 | Exactly what you think. Print a message to stderr |
---|
| 210 | """ |
---|
| 211 | print >>sys.stderr, msg |
---|
| 212 | |
---|
| 213 | |
---|
[b0581ac] | 214 | def make_new_name(names, prefix="name"): |
---|
| 215 | """ |
---|
| 216 | Generate an identifier not present in names by appending an integer to |
---|
| 217 | prefix. The new name is inserted into names and returned. |
---|
| 218 | """ |
---|
| 219 | i = 0 |
---|
| 220 | n = "%s%d" % (prefix,i) |
---|
| 221 | while n in names: |
---|
| 222 | i += 1 |
---|
| 223 | n = "%s%d" % (prefix,i) |
---|
| 224 | names.add(n) |
---|
| 225 | return n |
---|
[22c88c9] | 226 | |
---|
[2c51061] | 227 | def base_name(n): |
---|
| 228 | """ |
---|
| 229 | Extract a base name of the node to use for constructing a non-colliding |
---|
| 230 | name. This makes the composed topologies a little more readable. It's a |
---|
| 231 | single regexp, but the function name is more meaningful. |
---|
| 232 | """ |
---|
| 233 | return re.sub('\d+$', '',n) |
---|
| 234 | |
---|
| 235 | |
---|
| 236 | |
---|
[b0581ac] | 237 | def localize_names(top, names, marks): |
---|
| 238 | """ |
---|
| 239 | Take a topology and rename any substrates or elements that share a name |
---|
| 240 | with an existing computer or substrate. Keep the original name as a |
---|
| 241 | localized_name attribute. In addition, remap any constraints or interfaces |
---|
| 242 | that point to the old name over to the new one. Constraints are found in |
---|
| 243 | the marks dict, indexed by node name. Those constraints name attributes |
---|
| 244 | have already been converted to triples (node name, interface name, |
---|
| 245 | topology) so only the node name needs to be changed. |
---|
| 246 | """ |
---|
| 247 | sub_map = { } |
---|
| 248 | for s in top.substrates: |
---|
| 249 | s.set_attribute('localized_name', s.name) |
---|
| 250 | if s.name in names: |
---|
[2c51061] | 251 | sub_map[s.name] = n = make_new_name(names, base_name(s.name)) |
---|
[b0581ac] | 252 | s.name = n |
---|
| 253 | else: |
---|
| 254 | names.add(s.name) |
---|
| 255 | |
---|
| 256 | for e in [ e for e in top.elements if isinstance(e, topdl.Computer)]: |
---|
| 257 | e.set_attribute('localized_name', e.name) |
---|
| 258 | if e.name in names: |
---|
[2c51061] | 259 | nn= make_new_name(names, base_name(e.name)) |
---|
[7ee16b3] | 260 | for c in marks.get(e.name, []): |
---|
| 261 | c.name = nn |
---|
[b0581ac] | 262 | e.name = nn |
---|
| 263 | else: |
---|
| 264 | names.add(e.name) |
---|
| 265 | |
---|
| 266 | # Interface mapping. The list comprehension creates a list of |
---|
| 267 | # substrate names where each element in the list is replaced by the |
---|
| 268 | # entry in sub_map indexed by it if present and left alone otherwise. |
---|
| 269 | for i in e.interface: |
---|
| 270 | i.substrate = [ sub_map.get(ii, ii) for ii in i.substrate ] |
---|
| 271 | |
---|
[966c620] | 272 | def meet_constraints(candidates, provides, accepts, |
---|
| 273 | same_node=False, same_topo=False, multi_pair=False): |
---|
[b0581ac] | 274 | """ |
---|
| 275 | Try to meet all the constraints in candidates using the information in the |
---|
| 276 | provides and accepts dicts (which index constraints that provide or accept |
---|
| 277 | the given attribute). A constraint is met if it can be matched with another |
---|
| 278 | constraint that provides an attribute that the first constraint accepts. |
---|
| 279 | Only one match per pair is allowed, and we always prefer matching a |
---|
| 280 | required constraint to an unreqired one. If all the candidates can be |
---|
| 281 | matches, return True and return False otherwise. |
---|
| 282 | """ |
---|
| 283 | got_all = True |
---|
[7ee16b3] | 284 | node_match = { } |
---|
[b0581ac] | 285 | for c in candidates: |
---|
| 286 | if not c.match: |
---|
[6df4b11] | 287 | rmatch = None # Match to a required constraint |
---|
| 288 | umatch = None # Match to an unrequired constraint |
---|
[b0581ac] | 289 | for a in c.accepts: |
---|
| 290 | for can in provides.get(a,[]): |
---|
[966c620] | 291 | # A constraint cannot satisfy itself nor can it match |
---|
| 292 | # multiple times. |
---|
| 293 | if can != c and not can.match: |
---|
| 294 | # Unless same_node is true disallow nodes satisfying |
---|
| 295 | # their own constraints. |
---|
| 296 | if not same_node and can.name == c.name: |
---|
| 297 | continue |
---|
| 298 | # Unless same_topo is true, exclude nodes in the same |
---|
| 299 | # topology. |
---|
| 300 | if not same_topo and can.topology == c.topology: |
---|
| 301 | continue |
---|
[7ee16b3] | 302 | # Don't allow multiple matches between the same nodes |
---|
[966c620] | 303 | if not multi_pair and c.name in node_match and \ |
---|
[7ee16b3] | 304 | can.name in node_match[c.name]: |
---|
| 305 | continue |
---|
| 306 | # Now check that can also accepts c |
---|
[6df4b11] | 307 | for ca in can.accepts: |
---|
| 308 | if ca in c.provides: |
---|
| 309 | if can.required: rmatch = can |
---|
| 310 | else: umatch = can |
---|
| 311 | break |
---|
| 312 | |
---|
[b0581ac] | 313 | # Within providers, prefer matches against required |
---|
[6df4b11] | 314 | # composition points. |
---|
| 315 | if rmatch: |
---|
[b0581ac] | 316 | break |
---|
| 317 | # Within acceptance categories, prefer matches against required |
---|
| 318 | # composition points |
---|
[6df4b11] | 319 | if rmatch: |
---|
[b0581ac] | 320 | break |
---|
| 321 | |
---|
[6df4b11] | 322 | # Move the better match over to the match variable |
---|
| 323 | if rmatch: match = rmatch |
---|
| 324 | elif umatch: match = umatch |
---|
| 325 | else: match = None |
---|
| 326 | |
---|
| 327 | # Done checking all possible matches. Record the match or note an |
---|
| 328 | # unmatched candidate. |
---|
[b0581ac] | 329 | if match: |
---|
| 330 | match.match = c |
---|
| 331 | c.match = match |
---|
[7ee16b3] | 332 | # Note the match of the nodes |
---|
| 333 | for a, b in ((c.name, match.name), (match.name, c.name)): |
---|
| 334 | if a in node_match: node_match[a].append(b) |
---|
| 335 | else: node_match[a] = [ b] |
---|
[b0581ac] | 336 | else: |
---|
| 337 | got_all = False |
---|
| 338 | return got_all |
---|
[22c88c9] | 339 | |
---|
[b14b495] | 340 | def randomize_constraint_order(indexes): |
---|
| 341 | """ |
---|
| 342 | Randomly reorder the lists of constraints that provides and accepts hold. |
---|
| 343 | """ |
---|
| 344 | if not isinstance(indexes, tuple): |
---|
| 345 | indexes = (indexes,) |
---|
| 346 | |
---|
| 347 | for idx in indexes: |
---|
| 348 | for k in idx.keys(): |
---|
| 349 | random.shuffle(idx[k]) |
---|
| 350 | |
---|
[22c88c9] | 351 | def remote_ns2topdl(uri, desc, cert): |
---|
[b0581ac] | 352 | """ |
---|
| 353 | Call a remote service to convert the ns2 to topdl (and in fact all the way |
---|
| 354 | to a topdl.Topology. |
---|
| 355 | """ |
---|
[22c88c9] | 356 | |
---|
| 357 | req = { 'description' : { 'ns2description': desc }, } |
---|
| 358 | |
---|
[5f6c3af] | 359 | r = service_caller('Ns2Topdl')(uri, req, cert) |
---|
[22c88c9] | 360 | |
---|
| 361 | if r.has_key('Ns2TopdlResponseBody'): |
---|
| 362 | r = r['Ns2TopdlResponseBody'] |
---|
| 363 | ed = r.get('experimentdescription', None) |
---|
| 364 | if 'topdldescription' in ed: |
---|
| 365 | return topdl.Topology(**ed['topdldescription']) |
---|
| 366 | else: |
---|
| 367 | return None |
---|
| 368 | else: |
---|
| 369 | return None |
---|
| 370 | |
---|
[5e4025d] | 371 | def connect_composition_points(top, contraints, names): |
---|
[b0581ac] | 372 | """ |
---|
| 373 | top is a topology containing copies of all the topologies represented in |
---|
| 374 | the contsraints, flattened into one name space. This routine inserts the |
---|
| 375 | additional substrates required to interconnect the topology as described by |
---|
| 376 | the constraints. After the links are made, unused connection points are |
---|
| 377 | purged. |
---|
| 378 | """ |
---|
| 379 | done = set() |
---|
| 380 | for c in constraints: |
---|
| 381 | if c not in done and c.match: |
---|
| 382 | # c is an unprocessed matched constraint. Create a substrate |
---|
| 383 | # and attach it to the interfaces named by c and c.match. |
---|
| 384 | sn = make_new_name(names, "sub") |
---|
| 385 | s = topdl.Substrate(name=sn) |
---|
[89e2138] | 386 | |
---|
| 387 | # These are the nodes that need to be connected. Put an new |
---|
| 388 | # interface on each one and hook them to the new substrate. |
---|
[b0581ac] | 389 | for e in [ e for e in top.elements \ |
---|
[89e2138] | 390 | if isinstance(e, topdl.Computer) \ |
---|
| 391 | and e.name in (c.name, c.match.name)]: |
---|
| 392 | ii = make_new_name(set([x.name for x in e.interface]), "inf") |
---|
| 393 | e.interface.append( |
---|
| 394 | topdl.Interface(substrate=[sn], name=ii, element=e)) |
---|
| 395 | |
---|
| 396 | |
---|
| 397 | # c and c.match have been processed, so add them to the done set, |
---|
| 398 | # and add the substrate |
---|
| 399 | top.substrates.append(s) |
---|
[b0581ac] | 400 | done.add(c) |
---|
| 401 | done.add(c.match) |
---|
| 402 | |
---|
| 403 | top.incorporate_elements() |
---|
| 404 | |
---|
[5f6c3af] | 405 | def import_ns2_constraints(contents): |
---|
[ec9962b] | 406 | """ |
---|
| 407 | Contents is a list containing the lines of an annotated ns2 file. This |
---|
| 408 | routine extracts the constraint comment lines and convertes them into |
---|
| 409 | constraints in the namespace of the tcl experiment, as well as inserting |
---|
| 410 | them in the accepts and provides indices. |
---|
[5f6c3af] | 411 | |
---|
| 412 | Constraints are given in lines of the form |
---|
| 413 | name:required:provides:accepts |
---|
| 414 | where name is the name of the node in the current topology, if the second |
---|
| 415 | field is "required" this is a required constraint, a list of attributes |
---|
| 416 | that this connection point provides and a list that it accepts. The |
---|
| 417 | provides and accepts lists are comma-separated. The constraint is added to |
---|
| 418 | the marks dict under the name key and that dict is returned. |
---|
[ec9962b] | 419 | """ |
---|
[5f6c3af] | 420 | |
---|
[5e4025d] | 421 | const_re = re.compile("\s*#\s*COMPOSITION:\s*([^:]+:[^:]+:.*)") |
---|
| 422 | |
---|
[7ee16b3] | 423 | constraints = [ ] |
---|
[ec9962b] | 424 | for l in contents: |
---|
| 425 | m = const_re.search(l) |
---|
| 426 | if m: |
---|
[5f6c3af] | 427 | exp = re.sub('\s', '', m.group(1)) |
---|
| 428 | nn, r, p, a = exp.split(":") |
---|
[5e4025d] | 429 | if nn.find('(') != -1: |
---|
| 430 | # Convert array references into topdl names |
---|
| 431 | nn = re.sub('\\((\\d)\\)', '-\\1', nn) |
---|
[5f6c3af] | 432 | p = p.split(",") |
---|
| 433 | a = a.split(",") |
---|
[7ee16b3] | 434 | constraints.append(constraint(name=nn, required=(r == 'required'), |
---|
| 435 | provides=p, accepts=a)) |
---|
| 436 | return constraints |
---|
[5f6c3af] | 437 | |
---|
| 438 | def import_ns2_component(fn): |
---|
| 439 | """ |
---|
| 440 | Pull a composition component in from an ns2 description. The Constraints |
---|
| 441 | are parsed from the comments using import_ns2_constraints and the topology |
---|
| 442 | is created using a call to a fedd exporting the Ns2Topdl function. A |
---|
| 443 | topdl.Topology object rempresenting the component's topology and a dict |
---|
| 444 | mapping constraints from the components names to the conttraints is |
---|
| 445 | returned. If either the file read or the conversion fails, appropriate |
---|
| 446 | Exceptions are thrown. |
---|
| 447 | """ |
---|
| 448 | f = open(fn, "r") |
---|
| 449 | contents = [ l for l in f ] |
---|
| 450 | |
---|
| 451 | marks = import_ns2_constraints(contents) |
---|
| 452 | top = remote_ns2topdl(opts.url, "".join(contents), cert) |
---|
| 453 | if not top: |
---|
| 454 | raise RuntimeError("Cannot create topology from: %s" % fn) |
---|
| 455 | |
---|
| 456 | return (top, marks) |
---|
[ec9962b] | 457 | |
---|
[4b68c58] | 458 | def import_xml_component(fn): |
---|
[5e4025d] | 459 | """ |
---|
[7ee16b3] | 460 | Pull a component in from a topdl description. |
---|
[5e4025d] | 461 | """ |
---|
[7ee16b3] | 462 | return (topdl.topology_from_xml(filename=fn, top='experiment'), |
---|
| 463 | constraints_from_xml(filename=fn, top='constraints')) |
---|
[2e0a952] | 464 | |
---|
[7ee16b3] | 465 | def index_constraints(constraints, provides, accepts, names): |
---|
[5f6c3af] | 466 | """ |
---|
[5e4025d] | 467 | Add constraints to the provides and accepts indices based on the attributes |
---|
[7ee16b3] | 468 | of the contstraints. Also index by name. |
---|
[5f6c3af] | 469 | """ |
---|
| 470 | for c in constraints: |
---|
| 471 | for attr, dict in ((c.provides, provides), (c.accepts, accepts)): |
---|
| 472 | for a in attr: |
---|
| 473 | if a not in dict: dict[a] = [c] |
---|
| 474 | else: dict[a].append(c) |
---|
[7ee16b3] | 475 | if c.name in names: names[c.name].append(c) |
---|
| 476 | else: names[c.name]= [ c ] |
---|
[ec9962b] | 477 | |
---|
[5e4025d] | 478 | def get_suffix(fn): |
---|
[ec9962b] | 479 | """ |
---|
[5e4025d] | 480 | We get filename suffixes a couple places. It;s worth using the same code. |
---|
| 481 | This gets the shortest . separated suffix from a filename, or None. |
---|
[ec9962b] | 482 | """ |
---|
[5e4025d] | 483 | idx = fn.rfind('.') |
---|
[baf19c6] | 484 | if idx != -1: return fn[idx+1:] |
---|
[5e4025d] | 485 | else: return None |
---|
[ec9962b] | 486 | |
---|
| 487 | |
---|
[2c51061] | 488 | def output_composition(top, constraints, outfile=None, format=None, |
---|
| 489 | output_testbeds=False): |
---|
[5e4025d] | 490 | """ |
---|
| 491 | Output the composition to the file named by outfile (if any) in the format |
---|
| 492 | given by format if any. If both are None, output to stdout in topdl |
---|
| 493 | """ |
---|
[2c51061] | 494 | def xml_out(f, top, constraints, output_testbeds): |
---|
[5e4025d] | 495 | """ |
---|
| 496 | Output into topdl. Just call the topdl output, as the constraint |
---|
| 497 | attributes should be in the topology. |
---|
| 498 | """ |
---|
[4d4dde4] | 499 | print >>f, "<component>" |
---|
[89e2138] | 500 | if constraints: |
---|
| 501 | print >>f, "<constraints>" |
---|
| 502 | for c in constraints: |
---|
| 503 | print >>f, c.to_xml() |
---|
| 504 | print >>f, "</constraints>" |
---|
[5e4025d] | 505 | print >>f, topdl.topology_to_xml(comp, top='experiment') |
---|
[4d4dde4] | 506 | print >>f, "</component>" |
---|
[5e4025d] | 507 | |
---|
[2c51061] | 508 | def ns2_out(f, top, constraints, output_testbeds): |
---|
[5e4025d] | 509 | """ |
---|
| 510 | Reformat the constraint data structures into ns2 constraint comments |
---|
| 511 | and output the ns2 using the topdl routine. |
---|
| 512 | """ |
---|
| 513 | # Inner routines |
---|
| 514 | # Deal with the possibility that the single string name is still in the |
---|
| 515 | # constraint. |
---|
| 516 | def name_format(n): |
---|
| 517 | if isinstance(n, tuple): return n[0] |
---|
| 518 | else: return n |
---|
| 519 | |
---|
| 520 | |
---|
| 521 | # Format the required field back into a string. (ns2_out inner |
---|
| 522 | def required_format(x): |
---|
| 523 | if x: return "required" |
---|
| 524 | else: return "optional" |
---|
[2c51061] | 525 | |
---|
| 526 | def testbed_filter(e): |
---|
| 527 | if isinstance(e, topdl.Computer) and e.get_attribute('testbed'): |
---|
| 528 | return 'tb-set-node-testbed ${%s} "%s"' % \ |
---|
| 529 | (topdl.to_tcl_name(e.name), e.get_attribute('testbed')) |
---|
| 530 | else: |
---|
| 531 | return "" |
---|
| 532 | |
---|
| 533 | if output_testbeds: filters = [ testbed_filter ] |
---|
| 534 | else: filters = [ ] |
---|
[5e4025d] | 535 | |
---|
| 536 | # ns2_out main line |
---|
| 537 | for c in constraints: |
---|
| 538 | print >>f, "# COMPOSITION: %s:%s:%s:%s" % ( |
---|
| 539 | topdl.to_tcl_name(name_format(c.name)), |
---|
| 540 | required_format(c.required), ",".join(c.provides), |
---|
| 541 | ",".join(c.accepts)) |
---|
[2c51061] | 542 | print >>f, topdl.topology_to_ns2(top, filters=filters) |
---|
[5e4025d] | 543 | |
---|
| 544 | # Info to map from format to output routine. |
---|
| 545 | exporters = { |
---|
[4b68c58] | 546 | 'xml':xml_out, 'topdl':xml_out, |
---|
[5e4025d] | 547 | 'tcl': ns2_out, 'ns': ns2_out, |
---|
| 548 | } |
---|
| 549 | |
---|
| 550 | if format: |
---|
| 551 | # Explicit format set, use it |
---|
| 552 | if format in exporters: |
---|
| 553 | exporter = exporters[format] |
---|
| 554 | else: |
---|
| 555 | raise RuntimeError("Unknown format %s" % format) |
---|
| 556 | elif outfile: |
---|
| 557 | # Determine the format from the suffix (if any) |
---|
| 558 | s = get_suffix(outfile) |
---|
| 559 | if s and s in exporters: |
---|
| 560 | exporter = exporters[s] |
---|
| 561 | else: |
---|
| 562 | raise RuntimeError("Unknown format (suffix) %s" % outfile) |
---|
| 563 | else: |
---|
| 564 | # Both outfile and format are empty |
---|
[4b68c58] | 565 | exporter = xml_out |
---|
[b0581ac] | 566 | |
---|
[5e4025d] | 567 | # The actual output. Open the file, if any, and call the exporter |
---|
| 568 | if outfile: f = open(outfile, "w") |
---|
| 569 | else: f = sys.stdout |
---|
[2c51061] | 570 | exporter(f, top, constraints, output_testbeds) |
---|
[5e4025d] | 571 | if outfile: f.close() |
---|
[b0581ac] | 572 | |
---|
[5e4025d] | 573 | def import_components(files): |
---|
| 574 | """ |
---|
| 575 | Pull in the components. The input is a sequence of tuples where each tuple |
---|
| 576 | includes the name of the file to pull the component from and the number of |
---|
| 577 | copies to make of it. The routine to read a file is picked by its suffix. |
---|
| 578 | On completion, a tuple containing the number of components successfully |
---|
| 579 | read, a list of the topologies, a set of names in use accross all |
---|
| 580 | topologies (for inserting later names), the constraints extracted, and |
---|
| 581 | indexes mapping the attribute provices and accepted tinto the lists of |
---|
| 582 | constraints that provide or accept them is returned. |
---|
| 583 | """ |
---|
| 584 | importers = { |
---|
| 585 | 'tcl': import_ns2_component, |
---|
| 586 | 'ns': import_ns2_component, |
---|
[4b68c58] | 587 | 'xml':import_xml_component, |
---|
| 588 | 'topdl':import_xml_component, |
---|
[5e4025d] | 589 | } |
---|
| 590 | names = set() |
---|
| 591 | constraints = [ ] |
---|
| 592 | provides = { } |
---|
| 593 | accepts = { } |
---|
| 594 | components = 0 |
---|
| 595 | topos = [ ] |
---|
| 596 | |
---|
| 597 | for fn, cnt in files: |
---|
| 598 | try: |
---|
| 599 | s = get_suffix(fn) |
---|
| 600 | if s and s in importers: |
---|
[7ee16b3] | 601 | top, cons = importers[s](fn) |
---|
[5e4025d] | 602 | else: |
---|
| 603 | warn("Unknown suffix on file %s. Ignored" % fn) |
---|
| 604 | continue |
---|
| 605 | except service_error, e: |
---|
| 606 | warn("Remote error on %s: %s" % (fn, e)) |
---|
| 607 | continue |
---|
| 608 | except EnvironmentError, e: |
---|
| 609 | warn("Error on %s: %s" % (fn, e)) |
---|
| 610 | continue |
---|
[22c88c9] | 611 | |
---|
[5e4025d] | 612 | # Parsed the component and sonctraints, now work through the rest of |
---|
| 613 | # the pre-processing. We do this once per copy of the component |
---|
| 614 | # requested, cloning topologies and constraints each time. |
---|
| 615 | for i in range(0, cnt): |
---|
| 616 | components += 1 |
---|
| 617 | t = top.clone() |
---|
[7ee16b3] | 618 | c = copy.deepcopy(cons) |
---|
| 619 | marks = { } |
---|
[89e2138] | 620 | # Bind the constraints in m (the marks copy) to this topology |
---|
[7ee16b3] | 621 | for cc in c: |
---|
| 622 | cc.topology = t |
---|
| 623 | index_constraints(c, provides, accepts, marks) |
---|
| 624 | localize_names(t, names, marks) |
---|
| 625 | constraints.extend(c) |
---|
[5e4025d] | 626 | topos.append(t) |
---|
| 627 | |
---|
| 628 | return (components, topos, names, constraints, provides, accepts) |
---|
| 629 | |
---|
[9e38ded] | 630 | def parse_config(fn, opts): |
---|
| 631 | """ |
---|
| 632 | Pull configuration options from a file given in fn. These mirror the |
---|
| 633 | command line options. |
---|
| 634 | """ |
---|
| 635 | # These functions make up a jump table for parsing lines of config. |
---|
| 636 | # They're dispatched from directives below. |
---|
| 637 | def file(args, opts): |
---|
| 638 | i = args.find(',') |
---|
| 639 | if i == -1: opts.files.append((args, 1)) |
---|
| 640 | else: |
---|
| 641 | try: |
---|
| 642 | opts.files.append((args[0:i], int(args[i+1:]))) |
---|
| 643 | except ValueError: |
---|
| 644 | raise config_error("Cannot convert %s to an integer" % \ |
---|
| 645 | args[i+1:]) |
---|
| 646 | |
---|
| 647 | def url(args, opts): |
---|
| 648 | opts.url=args |
---|
| 649 | |
---|
| 650 | def output(args, opts): |
---|
| 651 | opts.outfile=args |
---|
| 652 | |
---|
| 653 | def format(args, opts): |
---|
| 654 | valid = ("xml", "topdl", "tcl", "ns") |
---|
| 655 | if args in valid: |
---|
| 656 | opts.format = args |
---|
| 657 | else: |
---|
| 658 | raise config_error("Bad format %s. Must be one of %s" % \ |
---|
| 659 | (args, valid)) |
---|
| 660 | |
---|
| 661 | def tb_out(args, opts): |
---|
| 662 | opts.output_testbeds = True |
---|
| 663 | |
---|
| 664 | def tb_add(args, opts): |
---|
| 665 | opts.add_testbeds = True |
---|
| 666 | |
---|
| 667 | def seed(args, opts): |
---|
| 668 | try: |
---|
| 669 | opts.seed = int(args) |
---|
| 670 | except ValueError: |
---|
| 671 | raise config_error("Cannot convert %s to an integer" % args) |
---|
| 672 | |
---|
| 673 | def lax(args, opts): |
---|
| 674 | opts.lax = True |
---|
| 675 | |
---|
| 676 | def same_node(args, opts): |
---|
| 677 | opts.same_node = True |
---|
| 678 | |
---|
| 679 | def same_topo(args, opts): |
---|
| 680 | opts.same_topo = True |
---|
| 681 | |
---|
| 682 | def same_pair(args, opts): |
---|
| 683 | opts.multi_pair = True |
---|
| 684 | |
---|
| 685 | directives = { |
---|
| 686 | 'file': file, |
---|
| 687 | 'multifile': file, |
---|
| 688 | 'output': output, |
---|
| 689 | 'url': url, |
---|
| 690 | 'format': format, |
---|
| 691 | 'output_testbeds': tb_out, |
---|
| 692 | 'add_testbeds': tb_add, |
---|
| 693 | 'seed': seed, |
---|
| 694 | 'lax': lax, |
---|
| 695 | 'same_node': same_node, |
---|
| 696 | 'same_topo': same_topo, |
---|
| 697 | 'same_pair': same_pair, |
---|
| 698 | } |
---|
| 699 | comment_re = re.compile('^\s*#') |
---|
| 700 | blank_re = re.compile('^\s*$') |
---|
| 701 | |
---|
| 702 | # Parse_config code begins |
---|
| 703 | f = open(fn, "r") |
---|
| 704 | try: |
---|
| 705 | for i, l in enumerate(f): |
---|
| 706 | if blank_re.match(l) or comment_re.match(l): |
---|
| 707 | continue |
---|
| 708 | |
---|
| 709 | if l.find('=') != -1: dir, args = l.split('=', 1) |
---|
| 710 | else: dir, args = (l, "") |
---|
| 711 | |
---|
| 712 | dir, args = (dir.strip(), args.strip()) |
---|
| 713 | |
---|
| 714 | try: |
---|
| 715 | if dir in directives: directives[dir](args, opts) |
---|
| 716 | else: file(dir, opts) |
---|
| 717 | except config_error, e: |
---|
| 718 | raise config_error("%s in line %d" % (e, i+1)) |
---|
| 719 | finally: |
---|
| 720 | if f: f.close() |
---|
| 721 | |
---|
| 722 | |
---|
[5e4025d] | 723 | # Main line begins |
---|
| 724 | parser = ComposeOptionParser() |
---|
[22c88c9] | 725 | |
---|
| 726 | opts, args = parser.parse_args() |
---|
| 727 | |
---|
[9e38ded] | 728 | if opts.config: |
---|
| 729 | try: |
---|
| 730 | parse_config(opts.config, opts) |
---|
| 731 | except EnvironmentError, e: |
---|
| 732 | sys.exit("Can't open configuration: %s" %e) |
---|
| 733 | except config_error, e: |
---|
| 734 | sys.exit(e) |
---|
| 735 | |
---|
[22c88c9] | 736 | if opts.cert: |
---|
| 737 | cert = opts.cert |
---|
| 738 | elif os.access(os.path.expanduser("~/.ssl/emulab.pem"), os.R_OK): |
---|
| 739 | cert = os.path.expanduser("~/.ssl/emulab.pem") |
---|
| 740 | else: |
---|
| 741 | cert = None |
---|
| 742 | |
---|
[b14b495] | 743 | random.seed(opts.seed) |
---|
| 744 | |
---|
[ec9962b] | 745 | files = opts.files |
---|
| 746 | files.extend([ (a, 1) for a in args]) |
---|
| 747 | |
---|
[5e4025d] | 748 | # Pull 'em in. |
---|
| 749 | components, topos, names, constraints, provides, accepts = \ |
---|
| 750 | import_components(files) |
---|
[22c88c9] | 751 | |
---|
[5e4025d] | 752 | # If more than one component was given, actually do the composition, otherwise |
---|
| 753 | # this is probably a format conversion. |
---|
| 754 | if components > 1: |
---|
| 755 | # Mix up the constraint indexes |
---|
| 756 | randomize_constraint_order((provides, accepts)) |
---|
| 757 | |
---|
| 758 | # Now the various components live in the same namespace and are marked with |
---|
| 759 | # their composition requirements. |
---|
| 760 | |
---|
| 761 | if not meet_constraints([c for c in constraints if c.required], |
---|
[966c620] | 762 | provides, accepts, opts.same_node, opts.same_topo, opts.multi_pair): |
---|
[6df4b11] | 763 | if opts.lax: |
---|
| 764 | warn("warning: could not meet all required constraints") |
---|
| 765 | else: |
---|
| 766 | sys.exit("Failed: could not meet all required constraints") |
---|
[5e4025d] | 767 | |
---|
| 768 | meet_constraints([ c for c in constraints if not c.match ], |
---|
[966c620] | 769 | provides, accepts, opts.same_node, opts.same_topo, opts.multi_pair) |
---|
[5e4025d] | 770 | |
---|
[023e79b] | 771 | # Add testbed attributes if requested |
---|
| 772 | if opts.add_testbeds: |
---|
| 773 | for i, t in enumerate(topos): |
---|
| 774 | for e in [ e for e in t.elements if isinstance(e, topdl.Computer)]: |
---|
| 775 | e.set_attribute('testbed', 'testbed%03d' % i) |
---|
| 776 | |
---|
[5e4025d] | 777 | # Make a topology containing all elements and substrates from components |
---|
| 778 | # that had matches. |
---|
| 779 | comp = topdl.Topology() |
---|
[89e2138] | 780 | for t in set([ c.topology for c in constraints if c.match]): |
---|
[5e4025d] | 781 | comp.elements.extend([e.clone() for e in t.elements]) |
---|
| 782 | comp.substrates.extend([s.clone() for s in t.substrates]) |
---|
| 783 | |
---|
| 784 | # Add substrates and connections corresponding to composition points |
---|
| 785 | connect_composition_points(comp, constraints, names) |
---|
| 786 | |
---|
[4d4dde4] | 787 | elif components == 1: |
---|
[5e4025d] | 788 | comp = topos[0] |
---|
[023e79b] | 789 | if opts.add_testbeds: |
---|
| 790 | for e in [ e for e in comp.elements if isinstance(e, topdl.Computer)]: |
---|
| 791 | e.set_attribute('testbed', 'testbed001') |
---|
| 792 | |
---|
[5e4025d] | 793 | else: |
---|
| 794 | sys.exit("Did not read any components.") |
---|
[b0581ac] | 795 | |
---|
[4d4dde4] | 796 | # Put out the composition with only the unmatched constraints |
---|
[5e4025d] | 797 | output_composition(comp, [c for c in constraints if not c.match], |
---|
[2c51061] | 798 | opts.outfile, opts.format, opts.output_testbeds) |
---|
[4b68c58] | 799 | |
---|
| 800 | sys.exit(0) |
---|