source: fedd/compose.py @ 2e0a952

axis_examplecompt_changesinfo-opsversion-3.01version-3.02
Last change on this file since 2e0a952 was 2e0a952, checked in by Ted Faber <faber@…>, 14 years ago

Take topdl input

  • Property mode set to 100644
File size: 12.9 KB
Line 
1#!/usr/local/bin/python
2
3import sys
4import re
5import os
6import random
7import copy
8
9from optparse import OptionParser, OptionValueError
10from federation.remote_service import service_caller
11from federation.service_error import service_error
12from federation import topdl
13
14class constraint:
15    def __init__(self, name=None, required=False, accepts=None, provides=None,
16            match=None):
17        self.name = name
18        self.required = required
19        self.accepts = accepts or []
20        self.provides = provides or []
21        self.match = match
22
23    def __str__(self):
24        return "%s:%s:%s:%s" % (self.name, self.required,
25                ",".join(self.provides), ",".join(self.accepts))
26
27def make_new_name(names, prefix="name"):
28    """
29    Generate an identifier not present in names by appending an integer to
30    prefix.  The new name is inserted into names and returned.
31    """
32    i = 0
33    n = "%s%d" % (prefix,i)
34    while n in names:
35        i += 1
36        n = "%s%d" % (prefix,i)
37    names.add(n)
38    return n
39
40def add_interfaces(top, mark):
41    """
42    Add unconnected interfaces to the nodes whose names are keys in the mark
43    dict.  As the interface is added change the name field of the contstraint
44    in mark into a tuple containing the node name, interface name, and topology
45    in which they reside.
46    """
47    for e in top.elements:
48        if e.name in mark:
49            con = mark[e.name]
50            ii = make_new_name(set([x.name for x in e.interface]), "inf")
51            e.interface.append(
52                    topdl.Interface(substrate=[], name=ii, element=e))
53            con.name = (e.name, ii, top)
54
55def localize_names(top, names, marks):
56    """
57    Take a topology and rename any substrates or elements that share a name
58    with an existing computer or substrate.  Keep the original name as a
59    localized_name attribute.  In addition, remap any constraints or interfaces
60    that point to the old name over to the new one.  Constraints are found in
61    the marks dict, indexed by node name.  Those constraints name attributes
62    have already been converted to triples (node name, interface name,
63    topology) so only the node name needs to be changed.
64    """
65    sub_map = { }
66    for s in top.substrates:
67        s.set_attribute('localized_name', s.name)
68        if s.name in names:
69            sub_map[s.name] = n = make_new_name(names, "substrate")
70            s.name = n
71        else:
72            names.add(s.name)
73
74    for e in [ e for e in top.elements if isinstance(e, topdl.Computer)]:
75        e.set_attribute('localized_name', e.name)
76        if e.name in names:
77            nn= make_new_name(names, "computer")
78            if e.name in marks:
79                n, i, t = marks[e.name].name
80                marks[e.name].name = (nn, i, t)
81            e.name = nn
82        else:
83            names.add(e.name)
84
85        # Interface mapping.  The list comprehension creates a list of
86        # substrate names where each element in the list is replaced by the
87        # entry in sub_map indexed by it if present and left alone otherwise.
88        for i in e.interface:
89            i.substrate = [ sub_map.get(ii, ii) for ii in i.substrate ]
90
91def meet_constraints(candidates, provides, accepts):
92    """
93    Try to meet all the constraints in candidates using the information in the
94    provides and accepts dicts (which index constraints that provide or accept
95    the given attribute). A constraint is met if it can be matched with another
96    constraint that provides an attribute that the first constraint accepts.
97    Only one match per pair is allowed, and we always prefer matching a
98    required constraint to an unreqired one.  If all the candidates can be
99    matches, return True and return False otherwise.
100    """
101    got_all = True
102    for c in candidates:
103        if not c.match:
104            match = None
105            for a in c.accepts:
106                for can in provides.get(a,[]):
107                    # A constraint cannot satisfy itself, nor do we allow loops
108                    # within the same topology.  This also excludes matched
109                    # constraints.
110                    if can.name != c.name and can.name[2] != c.name[2] and \
111                            not can.match:
112                        match = can
113                        # Within providers, prefer matches against required
114                        # composition points
115                        if can.required:
116                            break
117                # Within acceptance categories, prefer matches against required
118                # composition points
119                if match and match.required:
120                    break
121
122            # Done checking all possible matches.
123            if match:
124                match.match = c
125                c.match = match
126            else:
127                got_all = False
128    return got_all
129
130def randomize_constraint_order(indexes):
131    """
132    Randomly reorder the lists of constraints that provides and accepts hold.
133    """
134    if not isinstance(indexes, tuple):
135        indexes = (indexes,)
136
137    for idx in indexes:
138        for k in idx.keys():
139            random.shuffle(idx[k])
140
141def remote_ns2topdl(uri, desc, cert):
142    """
143    Call a remote service to convert the ns2 to topdl (and in fact all the way
144    to a topdl.Topology.
145    """
146
147    req = { 'description' : { 'ns2description': desc }, }
148
149    r = service_caller('Ns2Topdl')(uri, req, cert)
150
151    if r.has_key('Ns2TopdlResponseBody'):
152        r = r['Ns2TopdlResponseBody']
153        ed = r.get('experimentdescription', None)
154        if 'topdldescription' in ed:
155            return topdl.Topology(**ed['topdldescription'])
156        else:
157            return None
158    else:
159        return None
160
161def connect_composition_points(top, contraints):
162    """
163    top is a topology containing copies of all the topologies represented in
164    the contsraints, flattened into one name space.  This routine inserts the
165    additional substrates required to interconnect the topology as described by
166    the constraints.  After the links are made, unused connection points are
167    purged.
168    """
169    done = set()
170    for c in constraints:
171        if c not in done and c.match:
172            # c is an unprocessed matched constraint.  Create a substrate
173            # and attach it to the interfaces named by c and c.match.
174            sn = make_new_name(names, "sub")
175            s = topdl.Substrate(name=sn)
176            connected = 0
177            # Walk through all the computers in the topology
178            for e in [ e for e in top.elements \
179                    if isinstance(e, topdl.Computer)]:
180                # if the computer matches the computer and interface name in
181                # either c or c.match, connect it.  Once both computers have
182                # been found and connected, exit the loop walking through all
183                # computers.
184                for comp, inf in (c.name[0:2], c.match.name[0:2]):
185                    if e.name == comp:
186                        for i in e.interface:
187                            if i.name == inf:
188                                i.substrate.append(sn)
189                                connected += 1
190                                break
191                        break
192                # Connected both, so add the substrate to the topology
193                if connected == 2: 
194                    top.substrates.append(s)
195                    break
196            # c and c.match have been processed, so add them to the done set
197            done.add(c)
198            done.add(c.match)
199
200    # All interfaces with matched constraints have been connected.  Cull any
201    # interfaces unassigned to a substrate
202    for e in [ e for e in top.elements if isinstance(e, topdl.Computer)]:
203        if any([ not i.substrate for i in e.interface]):
204            e.interface = [ i for i in e.interface if i.substrate ]
205
206    top.incorporate_elements()
207
208def import_ns2_constraints(contents):
209    """
210    Contents is a list containing the lines of an annotated ns2 file.  This
211    routine extracts the constraint comment lines and convertes them into
212    constraints in the namespace of the tcl experiment, as well as inserting
213    them in the accepts and provides indices.
214
215    Constraints are given in lines of the form
216        name:required:provides:accepts
217    where name is the name of the node in the current topology, if the second
218    field is "required" this is a required constraint, a list of attributes
219    that this connection point provides and a list that it accepts.  The
220    provides and accepts lists are comma-separated.  The constraint is added to
221    the marks dict under the name key and that dict is returned.
222    """
223
224    marks = { }
225    for l in contents:
226        m = const_re.search(l)
227        if m:
228            exp = re.sub('\s', '', m.group(1))
229            nn, r, p, a = exp.split(":")
230            p = p.split(",")
231            a = a.split(",")
232            c = constraint(name=nn, required=(r == 'required'),
233                    provides=p, accepts=a)
234            marks[nn] = c
235    return marks
236
237def import_ns2_component(fn):
238    """
239    Pull a composition component in from an ns2 description.  The Constraints
240    are parsed from the comments using import_ns2_constraints and the topology
241    is created using a call to a fedd exporting the Ns2Topdl function.  A
242    topdl.Topology object rempresenting the component's topology and a dict
243    mapping constraints from the components names to the conttraints is
244    returned.  If either the file read or the conversion fails, appropriate
245    Exceptions are thrown.
246    """
247    f = open(fn, "r")
248    contents = [ l for l in f ]
249
250    marks = import_ns2_constraints(contents)
251    top = remote_ns2topdl(opts.url, "".join(contents), cert)
252    if not top:
253        raise RuntimeError("Cannot create topology from: %s" % fn)
254
255    return (top, marks)
256
257def import_topdl_component(fn):
258    top = topdl.topology_from_xml(filename=fn, top='experiment')
259    marks = { }
260    for e in [ e for e in top.elements if isinstance(e, topdl.Computer)]:
261        if e.get_attribute('composition_point'):
262            r = e.get_attribute('required') or 'false'
263            a = e.get_attribute('accepts') 
264            p = e.get_attribute('provides')
265            if a and p:
266                c = constraint(name=e.name, required=(r == 'required'),
267                    provides=p.split(','), accepts=a.split(','))
268                marks[e.name] = c
269    return (top, marks)
270
271def index_constraints(constraints, provides, accepts):
272    """
273    Add constraints to the provides and accepts indices based on what the
274    attributes of the contstraints.
275    """
276    for c in constraints:
277        for attr, dict in ((c.provides, provides), (c.accepts, accepts)):
278            for a in attr:
279                if a not in dict: dict[a] = [c]
280                else: dict[a].append(c)
281
282def multi_callback(option, opt_str, value, parser):
283    """
284    Parse a --multifile command line option.  The parameter is of the form
285    filename,count.  This splits the argument at the rightmost comma and
286    inserts the filename, count tuple into the "files" option.  It handles a
287    couple error cases, too.  This is an optparse.OptionParser callback.
288    """
289    idx = value.rfind(',')
290    if idx != -1:
291        try:
292            parser.values.files.append((value[0:idx], int(value[idx+1:])))
293        except ValueError, e:
294            raise OptionValueError("Can't convert %s to int in multifile (%s)" \
295                    % (value[idx+1:], value))
296    else:
297        raise OptionValueError("Bad format (need a comma) for multifile: %s" \
298                % value)
299
300
301
302# Main line begins
303
304const_re = re.compile("\s*#\s*COMPOSITION:\s*([^:]+:[^:]+:.*)")
305
306parser = OptionParser()
307parser.add_option('--url', dest='url', default="http://localhost:13235",
308        help='url of ns2 to topdl service')
309parser.add_option('--certfile', dest='cert', default=None,
310        help='Certificate to use as identity')
311parser.add_option('--seed', dest='seed', type='int', default=None,
312        help='Random number seed')
313parser.add_option('--multifile', dest='files', default=[], type='string', 
314        action='callback', callback=multi_callback, 
315        help="Include file multiple times")
316
317opts, args = parser.parse_args()
318
319if opts.cert:
320    cert = opts.cert
321elif os.access(os.path.expanduser("~/.ssl/emulab.pem"), os.R_OK):
322    cert = os.path.expanduser("~/.ssl/emulab.pem")
323else:
324    cert = None
325
326random.seed(opts.seed)
327
328files = opts.files
329files.extend([ (a, 1) for a in args])
330
331names = set()
332constraints = [ ]
333provides = { }
334accepts = { }
335imp = ( (('.tcl', '.ns'), import_ns2_component), 
336        (('.xml', '.topdl'), import_topdl_component),
337        )
338for fn, cnt in files:
339    try:
340        for suffix, importer in imp:
341            if fn.endswith(suffix):
342                top, marks = importer(fn)
343                break
344        else:
345            print >>sys.stderr, "Unknown suffix on file %s.  Ignored" % fn
346            continue
347    except service_error, e:
348        print >>sys.stderr, "Remote error on %s: %s" % (fn, e)
349        continue
350    except EnvironmentError, e:
351        print >>sys.stderr, "Error on %s: %s" % (fn, e)
352        continue
353
354    for i in range(0, cnt):
355        t = top.clone()
356        m = copy.deepcopy(marks)
357        index_constraints(m.values(), provides, accepts)
358        add_interfaces(t, m)
359        localize_names(t, names, m)
360        t.incorporate_elements()
361        constraints.extend(m.values())
362
363# Let the user know if they messed up on specifying constraints.
364if any([ not isinstance(c.name, tuple) for c in constraints]):
365    print >>sys.stderr, "nodes not found for constraints on %s" % \
366            ",".join([ c.name for c in constraints \
367            if isinstance(c.name, basestring)])
368    constraints = [ c for c in constraints if isinstance(c.name, tuple )]
369
370# Mix up the constraint indexes
371randomize_constraint_order((provides, accepts))
372
373# Now the various components live in the same namespace and are marked with
374# their composition requirements.
375
376if not meet_constraints([c for c in constraints if c.required], 
377        provides, accepts):
378    sys.exit("Could not meet all required constraints")
379meet_constraints([ c for c in constraints if not c.match ], provides, accepts)
380
381# Make a topology containing all elements and substrates from components that
382# had matches.
383comp = topdl.Topology()
384for t in set([ c.name[2] for c in constraints if c.match]):
385    comp.elements.extend([e.clone() for e in t.elements])
386    comp.substrates.extend([s.clone() for s in t.substrates])
387
388# Add substrates and connections corresponding to composition points
389connect_composition_points(comp, constraints)
390
391print topdl.topology_to_xml(comp, top='experiment')
Note: See TracBrowser for help on using the repository browser.