source: fedd/compose.py @ ec9962b

axis_examplecompt_changesinfo-opsversion-3.01version-3.02
Last change on this file since ec9962b was ec9962b, checked in by Ted Faber <faber@…>, 14 years ago

Make it simpler to specify multiple files on the command line.

  • Property mode set to 100644
File size: 11.5 KB
Line 
1#!/usr/local/bin/python
2
3import sys
4import re
5import os
6import random
7
8from optparse import OptionParser, OptionValueError
9from federation.remote_service import service_caller
10from federation import topdl
11
12class constraint:
13    def __init__(self, name=None, required=False, accepts=None, provides=None,
14            match=None):
15        self.name = name
16        self.required = required
17        self.accepts = accepts or []
18        self.provides = provides or []
19        self.match = match
20
21    def __str__(self):
22        return "%s:%s:%s:%s" % (self.name, self.required,
23                ",".join(self.provides), ",".join(self.accepts))
24
25
26def add_to_map(exp, mark, provides, accepts):
27    """
28    Create a constraint from a string of the form
29        name:required:provides:accepts
30    where name is the name of the node in the current topology, if the second
31    field is "required" this is a required constraint, a list of attributes
32    that this connection point provides and a list that it accepts.  The
33    provides and accepts lists are comma-separated.  Note that the string
34    passed in as exp should have whitespace removed. The constraint is added to
35    the mark dict under the name key and entries in the provides and accepts
36    are made to teh constraing for each attribute that it provides or accepts.
37    """
38    nn, r, p, a = exp.split(":")
39    p = p.split(",")
40    a = a.split(",")
41    c = constraint(name=nn, required=(r == 'required'), provides=p, accepts=a)
42    mark[nn] = c
43
44    for attr, dict in ((p, provides), (a, accepts)):
45        for a in attr:
46            if a not in dict: dict[a] = [c]
47            else: dict[a].append(c)
48
49def make_new_name(names, prefix="name"):
50    """
51    Generate an identifier not present in names by appending an integer to
52    prefix.  The new name is inserted into names and returned.
53    """
54    i = 0
55    n = "%s%d" % (prefix,i)
56    while n in names:
57        i += 1
58        n = "%s%d" % (prefix,i)
59    names.add(n)
60    return n
61
62def add_interfaces(top, mark):
63    """
64    Add unconnected interfaces to the nodes whose names are keys in the mark
65    dict.  As the interface is added change the name field of the contstraint
66    in mark into a tuple containing the node name, interface name, and topology
67    in which they reside.
68    """
69    for e in top.elements:
70        if e.name in mark:
71            con = mark[e.name]
72            ii = make_new_name(set([x.name for x in e.interface]), "inf")
73            e.interface.append(
74                    topdl.Interface(substrate=[], name=ii, element=e))
75            con.name = (e.name, ii, top)
76
77def localize_names(top, names, marks):
78    """
79    Take a topology and rename any substrates or elements that share a name
80    with an existing computer or substrate.  Keep the original name as a
81    localized_name attribute.  In addition, remap any constraints or interfaces
82    that point to the old name over to the new one.  Constraints are found in
83    the marks dict, indexed by node name.  Those constraints name attributes
84    have already been converted to triples (node name, interface name,
85    topology) so only the node name needs to be changed.
86    """
87    sub_map = { }
88    for s in top.substrates:
89        s.set_attribute('localized_name', s.name)
90        if s.name in names:
91            sub_map[s.name] = n = make_new_name(names, "substrate")
92            s.name = n
93        else:
94            names.add(s.name)
95
96    for e in [ e for e in top.elements if isinstance(e, topdl.Computer)]:
97        e.set_attribute('localized_name', e.name)
98        if e.name in names:
99            nn= make_new_name(names, "computer")
100            if e.name in marks:
101                n, i, t = marks[e.name].name
102                marks[e.name].name = (nn, i, t)
103            e.name = nn
104        else:
105            names.add(e.name)
106
107        # Interface mapping.  The list comprehension creates a list of
108        # substrate names where each element in the list is replaced by the
109        # entry in sub_map indexed by it if present and left alone otherwise.
110        for i in e.interface:
111            i.substrate = [ sub_map.get(ii, ii) for ii in i.substrate ]
112
113def meet_constraints(candidates, provides, accepts):
114    """
115    Try to meet all the constraints in candidates using the information in the
116    provides and accepts dicts (which index constraints that provide or accept
117    the given attribute). A constraint is met if it can be matched with another
118    constraint that provides an attribute that the first constraint accepts.
119    Only one match per pair is allowed, and we always prefer matching a
120    required constraint to an unreqired one.  If all the candidates can be
121    matches, return True and return False otherwise.
122    """
123    got_all = True
124    for c in candidates:
125        if not c.match:
126            match = None
127            for a in c.accepts:
128                for can in provides.get(a,[]):
129                    # A constraint cannot satisfy itself, nor do we allow loops
130                    # within the same topology.  This also excludes matched
131                    # constraints.
132                    if can.name != c.name and can.name[2] != c.name[2] and \
133                            not can.match:
134                        match = can
135                        # Within providers, prefer matches against required
136                        # composition points
137                        if can.required:
138                            break
139                # Within acceptance categories, prefer matches against required
140                # composition points
141                if match and match.required:
142                    break
143
144            # Done checking all possible matches.
145            if match:
146                match.match = c
147                c.match = match
148            else:
149                got_all = False
150    return got_all
151
152def randomize_constraint_order(indexes):
153    """
154    Randomly reorder the lists of constraints that provides and accepts hold.
155    """
156    if not isinstance(indexes, tuple):
157        indexes = (indexes,)
158
159    for idx in indexes:
160        for k in idx.keys():
161            random.shuffle(idx[k])
162
163def remote_ns2topdl(uri, desc, cert):
164    """
165    Call a remote service to convert the ns2 to topdl (and in fact all the way
166    to a topdl.Topology.
167    """
168
169    req = { 'description' : { 'ns2description': desc }, }
170
171    try:
172        r = service_caller('Ns2Topdl')(uri, req, cert)
173    except:
174        return None
175
176    if r.has_key('Ns2TopdlResponseBody'):
177        r = r['Ns2TopdlResponseBody']
178        ed = r.get('experimentdescription', None)
179        if 'topdldescription' in ed:
180            return topdl.Topology(**ed['topdldescription'])
181        else:
182            return None
183    else:
184        return None
185
186def connect_composition_points(top, contraints):
187    """
188    top is a topology containing copies of all the topologies represented in
189    the contsraints, flattened into one name space.  This routine inserts the
190    additional substrates required to interconnect the topology as described by
191    the constraints.  After the links are made, unused connection points are
192    purged.
193    """
194    done = set()
195    for c in constraints:
196        if c not in done and c.match:
197            # c is an unprocessed matched constraint.  Create a substrate
198            # and attach it to the interfaces named by c and c.match.
199            sn = make_new_name(names, "sub")
200            s = topdl.Substrate(name=sn)
201            connected = 0
202            # Walk through all the computers in the topology
203            for e in [ e for e in top.elements \
204                    if isinstance(e, topdl.Computer)]:
205                # if the computer matches the computer and interface name in
206                # either c or c.match, connect it.  Once both computers have
207                # been found and connected, exit the loop walking through all
208                # computers.
209                for comp, inf in (c.name[0:2], c.match.name[0:2]):
210                    if e.name == comp:
211                        for i in e.interface:
212                            if i.name == inf:
213                                i.substrate.append(sn)
214                                connected += 1
215                                break
216                        break
217                # Connected both, so add the substrate to the topology
218                if connected == 2: 
219                    top.substrates.append(s)
220                    break
221            # c and c.match have been processed, so add them to the done set
222            done.add(c)
223            done.add(c.match)
224
225    # All interfaces with matched constraints have been connected.  Cull any
226    # interfaces unassigned to a substrate
227    for e in [ e for e in top.elements if isinstance(e, topdl.Computer)]:
228        if any([ not i.substrate for i in e.interface]):
229            e.interface = [ i for i in e.interface if i.substrate ]
230
231    top.incorporate_elements()
232
233def import_tcl_constraints(marks, provides, accepts, contents):
234    """
235    Contents is a list containing the lines of an annotated ns2 file.  This
236    routine extracts the constraint comment lines and convertes them into
237    constraints in the namespace of the tcl experiment, as well as inserting
238    them in the accepts and provides indices.
239    """
240    for l in contents:
241        m = const_re.search(l)
242        if m:
243            add_to_map(re.sub('\s', '', m.group(1)), marks, provides, 
244                    accepts)
245
246
247def multi_callback(option, opt_str, value, parser):
248    """
249    Parse a --multifile command line option.  The parameter is of the form
250    filename,count.  This splits the argument at the rightmost comma and
251    inserts the filename, count tuple into the "files" option.  It handles a
252    couple error cases, too.  This is an optparse.OptionParser callback.
253    """
254    idx = value.rfind(',')
255    if idx != -1:
256        try:
257            parser.values.files.append((value[0:idx], int(value[idx+1:])))
258        except ValueError, e:
259            raise OptionValueError("Can't convert %s to int in multifile (%s)" \
260                    % (value[idx+1:], value))
261    else:
262        raise OptionValueError("Bad format (need a comma) for multifile: %s" \
263                % value)
264
265
266
267# Main line begins
268
269const_re = re.compile("\s*#\s*COMPOSITION:\s*([^:]+:[^:]+:.*)")
270
271parser = OptionParser()
272parser.add_option('--url', dest='url', default="http://localhost:13235",
273        help='url of ns2 to topdl service')
274parser.add_option('--certfile', dest='cert', default=None,
275        help='Certificate to use as identity')
276parser.add_option('--seed', dest='seed', type='int', default=None,
277        help='Random number seed')
278parser.add_option('--multifile', dest='files', default=[], type='string', 
279        action='callback', callback=multi_callback, 
280        help="Include file multiple times")
281
282opts, args = parser.parse_args()
283
284if opts.cert:
285    cert = opts.cert
286elif os.access(os.path.expanduser("~/.ssl/emulab.pem"), os.R_OK):
287    cert = os.path.expanduser("~/.ssl/emulab.pem")
288else:
289    cert = None
290
291random.seed(opts.seed)
292
293files = opts.files
294files.extend([ (a, 1) for a in args])
295
296comps = [ ]
297names = set()
298constraints = [ ]
299provides = { }
300accepts = { }
301for fn, cnt in files:
302    marks = { }
303    try:
304        f = open(fn, "r")
305        contents = [ l for l in f ]
306    except EnvironmentError, e:
307        print >>sys.stderr, "Error on %s: %s" % (fn, e)
308        continue
309
310    top = remote_ns2topdl(opts.url, "".join(contents), cert)
311    if not top:
312        sys.exit("Cannot create topology from: %s" % fn)
313
314    for i in range(0, cnt):
315        t = top.clone()
316        import_tcl_constraints(marks, provides, accepts, contents)
317        add_interfaces(t, marks)
318        localize_names(t, names, marks)
319        t.incorporate_elements()
320        constraints.extend(marks.values())
321        comps.append(t)
322
323# Let the user know if they messed up on specifying constraints.
324if any([ not isinstance(c.name, tuple) for c in constraints]):
325    print >>sys.stderr, "nodes not found for constraints on %s" % \
326            ",".join([ c.name for c in constraints \
327            if isinstance(c.name, basestring)])
328    constraints = [ c for c in constraints if isinstance(c.name, tuple )]
329
330# Mix up the constraint indexes
331randomize_constraint_order((provides, accepts))
332
333# Now the various components live in the same namespace and are marked with
334# their composition requirements.
335
336if not meet_constraints([c for c in constraints if c.required], 
337        provides, accepts):
338    sys.exit("Could not meet all required constraints")
339meet_constraints([ c for c in constraints if not c.match ], provides, accepts)
340
341# Make a topology containing all elements and substrates from components that
342# had matches.
343comp = topdl.Topology()
344for t in set([ c.name[2] for c in constraints if c.match]):
345    comp.elements.extend([e.clone() for e in t.elements])
346    comp.substrates.extend([s.clone() for s in t.substrates])
347
348# Add substrates and connections corresponding to composition points
349connect_composition_points(comp, constraints)
350
351print topdl.topology_to_xml(comp, top='experiment')
Note: See TracBrowser for help on using the repository browser.