source: fedd/fedd_image.py @ 8fbef04

Last change on this file since 8fbef04 was befb8e4, checked in by Ted Faber <faber@…>, 6 years ago

Hey, neato doesn't like line continuations any more....

  • Property mode set to 100755
File size: 14.6 KB
Line 
1#!/usr/bin/env python
2
3import sys
4import os
5import tempfile
6import subprocess
7import xml.parsers.expat
8
9from deter import fedid
10from deter import topdl
11
12from federation.proof import proof
13from federation.remote_service import service_caller
14from federation.client_lib import client_opts, exit_with_fault, RPCException, \
15        wrangle_standard_options, do_rpc, get_experiment_names, save_certfile,\
16        log_authentication
17from federation.proof import proof
18
19
20class image_opts(client_opts):
21    def __init__(self):
22        client_opts.__init__(self)
23        self.add_option("--experiment_cert", dest="exp_certfile",
24                action='callback', callback=self.expand_file, type='str',
25                help="experiment certificate file")
26        self.add_option("--experiment_name", dest="exp_name",
27                type="string", help="human readable experiment name")
28        self.add_option("--output", dest="outfile",
29                action='callback', callback=self.expand_file, type='str',
30                help="output image file")
31        self.add_option("--format", dest="format", type="choice", 
32                choices=("jpg", "png", "dot", "svg"),
33                help="Output file format override")
34        self.add_option("--program", dest="neato", default=None,
35                type="string",
36                help="Program compatible with dot (from graphviz) used to " + \
37                        "render image")
38        self.add_option("--labels", dest='labels', action='store_true',
39                default=True, help='Label nodes and edges')
40        self.add_option("--no_labels", dest='labels',
41                default=True, action='store_false',
42                help='Label nodes and edges')
43        self.add_option('--pixels', dest="pixels", default=None,
44                type="int",
45                help="Size of output in pixels (diagrams are square")
46        self.add_option("--file", dest="file", 
47                action='callback', callback=self.expand_file, type='str',
48                help="experiment description file")
49        self.add_option("--group", dest="group", action="append", default=[], 
50                help="Group nodes by this attribute")
51
52def make_subgraph(topelems, attrs):
53    """
54    Take a list of topdl elements (Computers and Substrates ) and partition
55    them into groups based on the value of the given attribute.  All computers
56    with the same value for the attribute are in a subgraph. A substrate is in
57    the subgraph if all its interfaces are, and in the "default" subgraph if
58    not.
59    """
60
61    def add_to_map(m, a, v):
62        """
63        Add this value to the list in m[a] if that list exists, or create it.
64        """
65        if a in m: m[a].append(v)
66        else: m[a] = [ v]
67
68    if not attrs: return { }
69
70    sg = { }
71    attr, rest = attrs[0], attrs[1:]
72    for e in topelems:
73        if isinstance(e, topdl.Computer):
74            add_to_map(sg, e.get_attribute(attr) or 'default', e)
75        elif isinstance(e, topdl.Substrate):
76            # A little hairy.  If the substrate itself has the attribute, use
77            # that.  Otherwise, make a set of the values of the given attribute
78            # for all the elements in the substrate's interfaces.  If that set
79            # has one element, put the substrate in that subgraph.  Otherwise
80            # put it in default.
81
82            sa = e.get_attribute(attr)
83            if sa is not None:
84                add_to_map(sg, sa, e)
85            else:
86                atts = set([i.element.get_attribute(attr) or 'default' \
87                        for i in e.interfaces])
88
89                if len(atts) == 1: add_to_map(sg, atts.pop(), e)
90                else: add_to_map(sg, 'default', e)
91
92    if rest:
93        for k in sg.keys():
94            sg[k] = make_subgraph(sg[k], rest)
95
96    return sg
97
98def output_subgraphs(sg, dotfile):
99
100    # Construct subgraphs featurning nodes from those graphs
101    for sn, nodes in sg.items():
102        # Output the computers and lans in subgraphs, if necessary.  Subgraphs
103        # treated as clusters are designated by "cluster" starting the name.
104        # (The default subgraph is not treated as a cluster.
105        if sn != 'default': 
106            print >>dotfile, 'subgraph "cluster-%s" {' % sn
107            print >>dotfile, 'label="%s"' % sn
108        else: 
109            print >>dotfile, 'subgraph {'
110        print >>dotfile, 'color="#08c0f8"'
111        print >>dotfile, 'clusterrank="local"'
112
113        if isinstance(nodes, dict): output_subgraphs(nodes, dotfile)
114        elif isinstance(nodes, list):
115            # For each node in the subgraph, output its representation
116            for i, n in enumerate(nodes):
117                line = ""
118                if isinstance(n, topdl.Computer):
119                    if n.name:
120                        line = '\t"%s" [shape=box,style=filled,' % n.name
121                    else:
122                        line = '\t"unnamed_node%d" [shape=box,style=filled,' % i
123                    line += 'color=black,fillcolor="#60f8c0",regular=1]'
124                elif isinstance(n, topdl.Substrate):
125                    line = '\t"%s" [shape=ellipse, style=filled,' % n.name
126                    line += 'color=black,fillcolor="#80c0f8",regular=1]'
127                print >>dotfile, line
128
129        print >>dotfile, '}'
130
131def gen_dot_topo(t, labels, dotfile, sg_attr):
132    """
133    Write a dot description of the topology in t, a topdl.Topology to the open
134    file dotfile.  the gen_image function below has already put some of that
135    file together. This function only generates the nodes and connections.  If
136    labels is true, label the nodes.
137    """
138
139
140    # The routine draws a circle for substrates with more than 2 interfaces
141    # connected to it, so this makes lists of each of those.
142    lans = [ s for s in t.substrates if len(s.interfaces) != 2]
143    links = [ s for s in t.substrates if len(s.interfaces) == 2]
144
145    # Break into subgraphs by the attribute of one's given
146    if sg_attr: sg = make_subgraph(t.elements + lans, sg_attr)
147    else: sg = { 'default': t.elements + lans }
148
149
150    output_subgraphs(sg, dotfile)
151
152    # Pull the edges out ot the lans and links
153    for l in lans:
154        for i in l.interfaces:
155            ip = i.get_attribute('ip4_address')
156            if labels and ip:
157                print >>dotfile, '\t"%s" -- "%s" [headlabel="%s"]' % \
158                        (l.name, i.element.name, ip)
159            else:
160                print >>dotfile, '\t"%s" -- "%s"' % \
161                        (l.name, i.element.name)
162
163    for l in links:
164        s, d = l.interfaces[0:2] 
165        sip = s.get_attribute('ip4_address')
166        dip = d.get_attribute('ip4_address')
167        if labels and sip and dip:
168            print >>dotfile, \
169                    ('\t"%s" -- "%s" [label="%s",taillabel="%s",' + \
170                    'headlabel="%s"]') % \
171                    (s.element.name, d.element.name, l.name,
172                        sip, dip)
173        else:
174            print >>dotfile, '\t"%s" -- "%s" ' % \
175                    (s.element.name, d.element.name)
176
177def gen_image(top, file, fmt, neato, labels, pix=None, sg_attr=None):
178    """
179    Create a dot formatted input file from the topdl.Topology in top and run
180    dot on it, storing the output in a file named file.  The image format it in
181    fmt, and it has to be one of the formats understood by dot or neato.
182    Labels is true if the graph should be noted with node names and addresses.
183    Pix sets the sixe of the output in pixels (square images).  Neato is a
184    user-specified graph drawing programs that takes neato/dot compaitble
185    inputs.  If no neato is specified, the code looks for a suitable
186    executable.  Return the command's return code.
187    """
188
189    # Open up a temporary file for dot to turn into a visualization.  This
190    # may raise an exception
191    df, dotname = tempfile.mkstemp(prefix='fedd_client', suffix=".dot")
192    dotfile = os.fdopen(df, 'w')
193
194    # Look for neato if the executable is unspecified.
195    if not neato:
196        # If a subgraph attribute has been specified, prefer dot to neato.  dot
197        # is more "natural" looking graphs, but neato segregates better.
198        if sg_attr: 
199            search_list = ['/usr/bin/dot', '/usr/local/bin/dot',
200                    '/usr/bin/neato', '/usr/local/bin/neato']
201        else:
202            search_list = ['/usr/bin/neato', '/usr/local/bin/neato', 
203                '/usr/bin/dot', '/usr/local/bin/dot']
204
205        for f in search_list:
206            if os.access(f, os.X_OK):
207                neato = f
208                break
209        else:
210            raise RuntimeError("Cannot find graph rendering program")
211
212    # Some arguments bases on size, outputformat, destination, etc.
213    cmd = [neato, '-Gsplines=true']
214    if fmt != 'dot': cmd.append('-T%s' % fmt)
215    if file:
216        cmd.append('-o')
217        cmd.append(file)
218    cmd.append(dotname)
219
220    nodes = len(top.elements)
221    if nodes < 10: size = 5
222    elif nodes < 50: size = 10
223    else: size = 18
224
225    if pix:
226        dpi = pix / size
227    else:
228        dpi = None
229
230    # Begin composing the graph description
231    print >>dotfile, "graph G {"
232    if dpi:
233        print >>dotfile, \
234            '\tgraph [size="%i,%i", dpi="%i", ratio=fill, compound="true"];' \
235            % (size, size, dpi)
236    else:
237        print >>dotfile, \
238            '\tgraph [size="%i,%i", ratio=fill, compound="true"];' \
239            % (size, size)
240
241    if labels:
242        print >>dotfile, '\tnode [fontname=arial,fontsize=9,label="\N"];'
243        print >>dotfile, '\tedge [fontname=arial,fontsize=9];\n'
244    else:
245        print >>dotfile, '\tnode [label=""];'
246
247    gen_dot_topo(top, labels, dotfile, sg_attr=sg_attr)
248    print >>dotfile, "}"
249    dotfile.close()
250
251    # Redirect the drawing program stderr, and draw
252    dev_null = open("/dev/null", "w")
253    rv = subprocess.call(cmd, stderr=dev_null)
254    os.remove(dotname)
255    dev_null.close()
256
257    return rv
258
259def extract_topo_from_message(msg, serialize):
260    """
261    Extract the topdl.Topology from either an Ns2Topdl or Info response.  In
262    the unlikely event that the message is misformatted, a RuntimeError is
263    raised.  If the overall script is only serializing messages, this routine
264    probably returns None (unless do_rpc is modified to return a valid response
265    when serializing only).
266    """
267    if 'experimentdescription' in msg:
268        if 'topdldescription' in msg['experimentdescription']:
269            exp = msg['experimentdescription']['topdldescription']
270            return topdl.Topology(**exp)
271        else:
272            raise RuntimeError("Bad response: could not translate")
273    elif serialize:
274        return None
275    else:
276        raise RuntimeError("Bad response. %s" % e.message)
277
278def get_experiment_topo(opts, cert, url):
279    """
280    Get the topology of an existing experiment from a running fedd.  That is,
281    pull the experimentdescription from an Info operation. Extract the
282    experiment identifier from opts and raise a RuntimeError if insufficient or
283    ambiguous information is present there.  The Topology is returned.
284    """
285
286    # Figure out what experiment to retrieve
287    if opts.exp_name and opts.exp_certfile:
288        raise RuntimeError("Only one of --experiment_cert and " + \
289                "--experiment_name permitted")
290    elif opts.exp_certfile:
291        exp_id = { 'fedid': fedid(file=opts.exp_certfile) }
292    elif opts.exp_name:
293        exp_id = { 'localname' : opts.exp_name }
294    else:
295        raise RuntimeError("specify one of --experiment_cert, " + \
296                "--experiment_name, or --file")
297
298    # Get the topology and return it
299    req = { 'experiment': exp_id }
300    resp_dict = do_rpc(req,
301            url, opts.transport, cert, opts.trusted, 
302            serialize_only=opts.serialize_only,
303            tracefile=opts.tracefile,
304            caller=service_caller('Info'), responseBody="InfoResponseBody")
305
306    p = proof.from_dict(resp_dict.get('proof', {}))
307    if p and opts.auth_log:
308        log_authentication(opts.auth_log, 'Image', 'succeeded', p)
309    return extract_topo_from_message(resp_dict, opts.serialize_only)
310
311def get_file_topo(opts, cert, url):
312    """
313    Parse a topology file or convert an ns2 file into a topdl.Topology.  XML
314    encodings of the topology can be converted completely locally, but ns2
315    files must be converted by a fedd.  The topology is returned or a
316    RuntimeError, RPCError, or EnvironmentError (if the file is unreadable) is
317    raised.
318    """
319    # Try the easy case first.  Note that if the file is unreadable rather than
320    # unparsable, this attempt will raise an EnvironmentError.
321    try:
322        return topdl.topology_from_xml(filename=opts.file, top='experiment')
323    except xml.parsers.expat.ExpatError, e:
324        # Print an error message if debugging
325        if opts.debug > 1: print >>sys.stderr, "XML parse failed: %s" %e
326
327    # If we get here the parser above failed and we assume that the file is
328    # ns2/tcl.  Call out to the converter.  The exceptions above can come out
329    # of here, though EnvironmentErrors are unlikely.
330    if not cert:
331        raise RuntimeError("Couldn't find an identity certificate, " + \
332                "RPC (Ns2Topdl) needed.")
333
334    contents = "".join([l for l in open(opts.file, "r")])
335    msg = { 'description': { 'ns2description': contents }, }
336    if opts.debug > 1: print >>sys.stderr, msg
337    resp_dict = do_rpc(msg, 
338            url, opts.transport, cert, opts.trusted, 
339            serialize_only=opts.serialize_only,
340            tracefile=opts.tracefile,
341            caller=service_caller('Ns2Topdl'),
342            responseBody="Ns2TopdlResponseBody")
343
344    return extract_topo_from_message(resp_dict, opts.serialize_only)
345
346parser = image_opts()
347(opts, args) = parser.parse_args()
348
349url = None
350
351# Note that if we're converting a local topdl file, the absence of a cert or
352# abac directory may not be a big deal.  It's checked if needed below
353try:
354    cert, fid, url = wrangle_standard_options(opts)
355except RuntimeError, e:
356    cert = None
357    print >>sys.stderr, "Warning: %s" % e
358
359
360# Figure out where output is going and in what format.  If format is given and
361# understood, that's the output format.  Otherwise we try the last three
362# letters of the file as a suffix.  If all that fails, we fall back to
363# dot/neato format.
364if opts.format and opts.outfile:
365    fmt = opts.format
366    file = opts.outfile
367elif not opts.format and opts.outfile:
368    fmt = opts.outfile[-3:]
369    if fmt not in ("png", "jpg", "dot", "svg"):
370        print >>sys.stderr, \
371                "Unexpected file suffix (%s) and no format given, using %s" % \
372                (fmt, "dot")
373        fmt = "dot"
374    file = opts.outfile
375elif opts.format and not opts.outfile:
376    fmt = opts.format
377    file = None
378else:
379    fmt="dot"
380    file = None
381
382# Get the topology from a file or an experiment
383if not opts.file:
384    if not cert:
385        sys.exit("Couldn't find an identity certificate, RPC (Info) needed.")
386    try:
387        top = get_experiment_topo(opts, cert, url)
388    except RPCException, e:
389        exit_with_fault(e, 'image', opts)
390    except RuntimeError, e:
391        sys.exit("%s" % e)
392else:
393    if opts.exp_name or opts.exp_certfile:
394        print >>sys.stderr, \
395                "Ignoring experiment name or certificate, reading %s" \
396                % opts.file
397    try:
398        top = get_file_topo(opts, cert, url)
399    except RPCException, e:
400        print >>sys.stderr, "Cannot extract a topology from %s" % opts.file
401        exit_with_fault(e, 'image', opts)
402    except RuntimeError, e:
403        sys.exit("Cannot extract a topology from %s: %s" % (opts.file, e))
404    except EnvironmentError, e:
405        sys.exit("Cannot extract a topology from %s: %s" % (opts.file, e))
406
407    if not top:
408        sys.exit("Cannot extract a topology from %s" % opts.file)
409
410# and make an image
411if not opts.serialize_only: 
412    try:
413        if gen_image(top, file, fmt, opts.neato, opts.labels, opts.pixels,
414                opts.group) !=0:
415            sys.exit("Error rendering graph (subcommand returned non-0)" +
416                    "  Check output, this may be a non-fatal graphviz warning")
417    except EnvironmentError, e:
418        sys.exit("Failed to draw graph: %s" %e)
419    except RuntimeError, e:
420        sys.exit("Failed to draw graph: %s" %e)
421else:
422    sys.exit(0)
Note: See TracBrowser for help on using the repository browser.