Changeset 5e4025d


Ignore:
Timestamp:
May 17, 2010 10:07:48 AM (14 years ago)
Author:
Ted Faber <faber@…>
Branches:
axis_example, compt_changes, info-ops, master, version-3.01, version-3.02
Children:
baf19c6
Parents:
4a53c72
Message:

A lot of changes. Some internal refactoring for neatness, and them multiple input and output formats.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • fedd/compose.py

    r4a53c72 r5e4025d  
    2424        return "%s:%s:%s:%s" % (self.name, self.required,
    2525                ",".join(self.provides), ",".join(self.accepts))
     26
     27       
     28class ComposeOptionParser(OptionParser):
     29    """
     30    This class encapsulates the options to this script in one place.  It also
     31    holds the callback for the multifile choice.
     32    """
     33    def __init__(self):
     34        OptionParser.__init__(self)
     35        self.add_option('--url', dest='url', default="http://localhost:13235",
     36                help='url of ns2 to topdl service')
     37        self.add_option('--certfile', dest='cert', default=None,
     38                help='Certificate to use as identity')
     39        self.add_option('--seed', dest='seed', type='int', default=None,
     40                help='Random number seed')
     41        self.add_option('--multifile', dest='files', default=[], type='string',
     42                action='callback', callback=self.multi_callback,
     43                help="Include file multiple times")
     44        self.add_option('--output', dest='outfile', default=None,
     45                help='Output file name')
     46        self.add_option('--format', dest='format', default="xml",
     47                help='Output file format')
     48
     49    @staticmethod
     50    def multi_callback(option, opt_str, value, parser):
     51        """
     52        Parse a --multifile command line option.  The parameter is of the form
     53        filename,count.  This splits the argument at the rightmost comma and
     54        inserts the filename, count tuple into the "files" option.  It handles
     55        a couple error cases, too.
     56        """
     57        idx = value.rfind(',')
     58        if idx != -1:
     59            try:
     60                parser.values.files.append((value[0:idx], int(value[idx+1:])))
     61            except ValueError, e:
     62                raise OptionValueError(
     63                        "Can't convert %s to int in multifile (%s)" % \
     64                                (value[idx+1:], value))
     65        else:
     66            raise OptionValueError(
     67                    "Bad format (need a comma) for multifile: %s" % value)
     68
     69def warn(msg):
     70    """
     71    Exactly what you think.  Print a message to stderr
     72    """
     73    print >>sys.stderr, msg
     74
    2675
    2776def make_new_name(names, prefix="name"):
     
    159208        return None
    160209
    161 def connect_composition_points(top, contraints):
     210def connect_composition_points(top, contraints, names):
    162211    """
    163212    top is a topology containing copies of all the topologies represented in
     
    222271    """
    223272
     273    const_re = re.compile("\s*#\s*COMPOSITION:\s*([^:]+:[^:]+:.*)")
     274
    224275    marks = { }
    225276    for l in contents:
     
    228279            exp = re.sub('\s', '', m.group(1))
    229280            nn, r, p, a = exp.split(":")
     281            if nn.find('(') != -1:
     282                # Convert array references into topdl names
     283                nn = re.sub('\\((\\d)\\)', '-\\1', nn)
    230284            p = p.split(",")
    231285            a = a.split(",")
     
    234288            marks[nn] = c
    235289    return marks
     290
     291def add_constraint_attributes(top, marks):
     292    """
     293    Take the constraints in marks and add attributes representing them to the
     294    nodes in top.
     295    """
     296    for e in [e for e in top.elements if isinstance(e, topdl.Computer)]:
     297        if e.name in marks and not e.get_attribute('composition_point'):
     298            c = marks[e.name]
     299            e.set_attribute('composition_point', 'true')
     300            e.set_attribute('required', '%s' % c.required)
     301            e.set_attribute('provides', ",".join(c.provides))
     302            e.set_attribute('accepts', ",".join(c.accepts))
     303
     304def remove_constraint_attributes(top, marks):
     305    """
     306    Remove constraint attributes that match the ones in the dict from the
     307    topology.
     308    """
     309    for e in [e for e in top.elements \
     310            if isinstance(e, topdl.Computer) and e.name in marks]:
     311        e.remove_attribute('composition_point')
     312        e.remove_attribute('required')
     313        e.remove_attribute('provides')
     314        e.remove_attribute('accepts')
    236315
    237316def import_ns2_component(fn):
     
    252331    if not top:
    253332        raise RuntimeError("Cannot create topology from: %s" % fn)
     333    add_constraint_attributes(top, marks)
    254334
    255335    return (top, marks)
    256336
    257337def import_topdl_component(fn):
     338    """
     339    Pull a component in from a topdl description.  The topology generation is
     340    straightforward and the constraints are pulled from the attributes of
     341    Computer nodes in the experiment.  The compisition_point attribute being
     342    true marks a node as a composition point.  The required, provides and
     343    accepts attributes map directly into those constraint fields.  The dict of
     344    constraints and the topology are returned.
     345    """
    258346    top = topdl.topology_from_xml(filename=fn, top='experiment')
    259347    marks = { }
     
    271359def index_constraints(constraints, provides, accepts):
    272360    """
    273     Add constraints to the provides and accepts indices based on what the
    274     attributes of the contstraints.
     361    Add constraints to the provides and accepts indices based on the attributes
     362    of the contstraints.
    275363    """
    276364    for c in constraints:
     
    280368                else: dict[a].append(c)
    281369
    282 def multi_callback(option, opt_str, value, parser):
    283     """
    284     Parse a --multifile command line option.  The parameter is of the form
    285     filename,count.  This splits the argument at the rightmost comma and
    286     inserts the filename, count tuple into the "files" option.  It handles a
    287     couple error cases, too.  This is an optparse.OptionParser callback.
    288     """
    289     idx = value.rfind(',')
    290     if idx != -1:
     370def get_suffix(fn):
     371    """
     372    We get filename suffixes a couple places.  It;s worth using the same code.
     373    This gets the shortest . separated suffix from a filename, or None.
     374    """
     375    idx = fn.rfind('.')
     376    if idx != -1: return [idx+1:]
     377    else: return None
     378
     379
     380def output_composition(top, constraints, outfile=None, format=None):
     381    """
     382    Output the composition to the file named by outfile (if any) in the format
     383    given by format if any.  If both are None, output to stdout in topdl
     384    """
     385    def topdl_out(f, top, constraints):
     386        """
     387        Output into topdl.  Just call the topdl output, as the constraint
     388        attributes should be in the topology.
     389        """
     390        print >>f, topdl.topology_to_xml(comp, top='experiment')
     391
     392    def ns2_out(f, top, constraints):
     393        """
     394        Reformat the constraint data structures into ns2 constraint comments
     395        and output the ns2 using the topdl routine.
     396        """
     397        # Inner routines
     398        # Deal with the possibility that the single string name is still in the
     399        # constraint.
     400        def name_format(n):
     401            if isinstance(n, tuple): return n[0]
     402            else: return n
     403           
     404           
     405        # Format the required field back into a string. (ns2_out inner
     406        def required_format(x):
     407            if x: return "required"
     408            else: return "optional"
     409       
     410        # ns2_out main line
     411        for c in constraints:
     412            print >>f, "# COMPOSITION: %s:%s:%s:%s" % (
     413                    topdl.to_tcl_name(name_format(c.name)),
     414                    required_format(c.required), ",".join(c.provides),
     415                    ",".join(c.accepts))
     416        print >>f, topdl.topology_to_ns2(top)
     417
     418    # Info to map from format to output routine. 
     419    exporters = {
     420            'xml':topdl_out, 'topdl':topdl_out,
     421            'tcl': ns2_out, 'ns': ns2_out,
     422            }
     423
     424    if format:
     425        # Explicit format set, use it
     426        if format in exporters:
     427            exporter = exporters[format]
     428        else:
     429            raise RuntimeError("Unknown format %s" % format)
     430    elif outfile:
     431        # Determine the format from the suffix (if any)
     432        s = get_suffix(outfile)
     433        if s and s in exporters:
     434            exporter = exporters[s]
     435        else:
     436            raise RuntimeError("Unknown format (suffix) %s" % outfile)
     437    else:
     438        # Both outfile and format are empty
     439        exporter = topdl_out
     440
     441    # The actual output.  Open the file, if any, and call the exporter
     442    if outfile: f = open(outfile, "w")
     443    else: f = sys.stdout
     444    exporter(f, top, constraints)
     445    if outfile: f.close()
     446
     447def import_components(files):
     448    """
     449    Pull in the components.  The input is a sequence of tuples where each tuple
     450    includes the name of the file to pull the component from and the number of
     451    copies to make of it.  The routine to read a file is picked by its suffix.
     452    On completion, a tuple containing the number of components successfully
     453    read, a list of the topologies, a set of names in use accross all
     454    topologies (for inserting later names), the constraints extracted, and
     455    indexes mapping the attribute provices and accepted tinto the lists of
     456    constraints that provide or accept them is returned.
     457    """
     458    importers = {
     459            'tcl': import_ns2_component,
     460            'ns': import_ns2_component,
     461            'xml':import_topdl_component,
     462            'topdl':import_topdl_component,
     463            }
     464    names = set()
     465    constraints = [ ]
     466    provides = { }
     467    accepts = { }
     468    components = 0
     469    topos = [ ]
     470
     471    for fn, cnt in files:
     472        top = None
    291473        try:
    292             parser.values.files.append((value[0:idx], int(value[idx+1:])))
    293         except ValueError, e:
    294             raise OptionValueError("Can't convert %s to int in multifile (%s)" \
    295                     % (value[idx+1:], value))
    296     else:
    297         raise OptionValueError("Bad format (need a comma) for multifile: %s" \
    298                 % value)
    299 
    300 
     474            s = get_suffix(fn)
     475            if s and s in importers:
     476                top, marks = importers[s](fn)
     477            else:
     478                warn("Unknown suffix on file %s.  Ignored" % fn)
     479                continue
     480        except service_error, e:
     481            warn("Remote error on %s: %s" % (fn, e))
     482            continue
     483        except EnvironmentError, e:
     484            warn("Error on %s: %s" % (fn, e))
     485            continue
     486
     487        # Parsed the component and sonctraints, now work through the rest of
     488        # the pre-processing.  We do this once per copy of the component
     489        # requested, cloning topologies and constraints each time.
     490        for i in range(0, cnt):
     491            components += 1
     492            t = top.clone()
     493            m = copy.deepcopy(marks)
     494            index_constraints(m.values(), provides, accepts)
     495            add_interfaces(t, m)
     496            localize_names(t, names, m)
     497            t.incorporate_elements()
     498            constraints.extend(m.values())
     499            topos.append(t)
     500
     501    return (components, topos, names, constraints, provides, accepts)
    301502
    302503# Main line begins
    303 
    304 const_re = re.compile("\s*#\s*COMPOSITION:\s*([^:]+:[^:]+:.*)")
    305 
    306 parser = OptionParser()
    307 parser.add_option('--url', dest='url', default="http://localhost:13235",
    308         help='url of ns2 to topdl service')
    309 parser.add_option('--certfile', dest='cert', default=None,
    310         help='Certificate to use as identity')
    311 parser.add_option('--seed', dest='seed', type='int', default=None,
    312         help='Random number seed')
    313 parser.add_option('--multifile', dest='files', default=[], type='string',
    314         action='callback', callback=multi_callback,
    315         help="Include file multiple times")
     504parser = ComposeOptionParser()
    316505
    317506opts, args = parser.parse_args()
     
    329518files.extend([ (a, 1) for a in args])
    330519
    331 names = set()
    332 constraints = [ ]
    333 provides = { }
    334 accepts = { }
    335 imp = ( (('.tcl', '.ns'), import_ns2_component),
    336         (('.xml', '.topdl'), import_topdl_component),
    337         )
    338 for fn, cnt in files:
    339     try:
    340         for suffix, importer in imp:
    341             if fn.endswith(suffix):
    342                 top, marks = importer(fn)
    343                 break
    344         else:
    345             print >>sys.stderr, "Unknown suffix on file %s.  Ignored" % fn
    346             continue
    347     except service_error, e:
    348         print >>sys.stderr, "Remote error on %s: %s" % (fn, e)
    349         continue
    350     except EnvironmentError, e:
    351         print >>sys.stderr, "Error on %s: %s" % (fn, e)
    352         continue
    353 
    354     for i in range(0, cnt):
    355         t = top.clone()
    356         m = copy.deepcopy(marks)
    357         index_constraints(m.values(), provides, accepts)
    358         add_interfaces(t, m)
    359         localize_names(t, names, m)
    360         t.incorporate_elements()
    361         constraints.extend(m.values())
     520# Pull 'em in.
     521components, topos, names, constraints, provides, accepts = \
     522        import_components(files)
    362523
    363524# Let the user know if they messed up on specifying constraints.
    364525if any([ not isinstance(c.name, tuple) for c in constraints]):
    365     print >>sys.stderr, "nodes not found for constraints on %s" % \
     526    warn("nodes not found for constraints on %s" % \
    366527            ",".join([ c.name for c in constraints \
    367             if isinstance(c.name, basestring)])
     528            if isinstance(c.name, basestring)]))
    368529    constraints = [ c for c in constraints if isinstance(c.name, tuple )]
    369530
    370 # Mix up the constraint indexes
    371 randomize_constraint_order((provides, accepts))
    372 
    373 # Now the various components live in the same namespace and are marked with
    374 # their composition requirements.
    375 
    376 if not meet_constraints([c for c in constraints if c.required],
    377         provides, accepts):
    378     sys.exit("Could not meet all required constraints")
    379 meet_constraints([ c for c in constraints if not c.match ], provides, accepts)
    380 
    381 # Make a topology containing all elements and substrates from components that
    382 # had matches.
    383 comp = topdl.Topology()
    384 for t in set([ c.name[2] for c in constraints if c.match]):
    385     comp.elements.extend([e.clone() for e in t.elements])
    386     comp.substrates.extend([s.clone() for s in t.substrates])
    387 
    388 # Add substrates and connections corresponding to composition points
    389 connect_composition_points(comp, constraints)
    390 
    391 print topdl.topology_to_xml(comp, top='experiment')
     531# If more than one component was given, actually do the composition, otherwise
     532# this is probably a format conversion.
     533if components > 1:
     534    # Mix up the constraint indexes
     535    randomize_constraint_order((provides, accepts))
     536
     537    # Now the various components live in the same namespace and are marked with
     538    # their composition requirements.
     539
     540    if not meet_constraints([c for c in constraints if c.required],
     541            provides, accepts):
     542        sys.exit("Could not meet all required constraints")
     543
     544    meet_constraints([ c for c in constraints if not c.match ],
     545            provides, accepts)
     546
     547    # Make a topology containing all elements and substrates from components
     548    # that had matches.
     549    comp = topdl.Topology()
     550    for t in set([ c.name[2] for c in constraints if c.match]):
     551        comp.elements.extend([e.clone() for e in t.elements])
     552        comp.substrates.extend([s.clone() for s in t.substrates])
     553
     554    # Add substrates and connections corresponding to composition points
     555    connect_composition_points(comp, constraints, names)
     556
     557    # Remove composition attributes for satisfied constraints
     558    remove_constraint_attributes(comp, dict(
     559        [(c.name[0], c) for c in constraints if c.match]))
     560elif topos == 1:
     561    comp = topos[0]
     562else:
     563    sys.exit("Did not read any components.")
     564
     565# Put out the composition with only the unmatched constriaints
     566output_composition(comp, [c for c in constraints if not c.match],
     567        opts.outfile, opts.format)
Note: See TracChangeset for help on using the changeset viewer.