source: fedkit/fed-tun.pl @ 37a809f

Last change on this file since 37a809f was 37a809f, checked in by Ted Faber <faber@…>, 10 years ago

ssh 6.5 parsing workaround
  • Property mode set to 100755
File size: 14.8 KB
Line 
1#!/usr/bin/perl -w
2
3# Kevin Lahey, lahey@isi.edu
4# July 11, 2007
5
6# Set up ssh tunnel infrastructure for federation:
7#
8# * Parse the configuration file provided.
9#
10# * Figure out whether we're the initiator or the reciever;  if we're
11#   the receiver, we just need to set up ssh keys and exit.
12#
13# * Pick out the experimental interface, remove the IP address
14#
15# * Create a layer 2 ssh tunnel, set up bridging, and hang loose.
16#
17
18use strict;
19use Getopt::Std;
20use POSIX qw(strftime);
21use Sys::Hostname;
22use IO::File;
23use File::Copy;
24
25my $IFCONFIG = "/sbin/ifconfig";
26my $TMCC = "/usr/local/etc/emulab/tmcc";
27my $FINDIF = "/usr/local/etc/emulab/findif";
28# If a special version of ssh is required, it will be installed in
29# /usr/local/bin/ssh.  Otherwise use the usual one.
30my $SSH = -x "/usr/local/bin/ssh" ? "/usr/local/bin/ssh" : "/usr/bin/ssh";
31my $NC = "/usr/bin/nc";
32my $SSH_PORT = 22;
33my $ROUTE_GET = "/sbin/route get";  # XXX:  works on FreeBSD, but should
34                                    #       should be 'ip route get' for Linux
35my $sshd_config = "/etc/ssh/sshd_config";    # Probably should be a param
36
37# Ports that are forwarded between testbeds
38my $TMCD_PORT = 7777;
39my $SMBFS_PORT = 139;
40my $PUBSUB_PORT = 16505;
41my $SEER_PORT = 16606;
42
43
44my $remote_pubsub_port = $PUBSUB_PORT - 1;  # There will be a local
45                                            # pubsubd running, so we
46                                            # dodge the port on the
47                                            # remote tunnel node.
48die "Cannot exec $SSH" unless -x $SSH;
49
50sub setup_bridging;
51sub setup_tunnel_cfg;
52sub parse_config;
53
54# Option use is as follows:
55#    -f         filename containing config file
56#    -d         turn on debugging output
57#    -r         remotely invoked
58#               (if remotely invoked the last two args are the tun interface
59#               number and the remote address of the tunnel)
60
61my $usage = "Usage: fed-tun.pl [-r] [-d] [-f config-filename] [count addr]\n";
62
63my %opts;       # Note that this is used for both getops and the options file
64my @expected_opts = qw(active tunnelcfg bossname fsname type
65                       peer pubkeys privkeys);
66my $filename;
67my $remote;
68my $debug = 1;
69my $count;
70my $addr;
71my $active;
72my $type;
73my $tunnelcfg;
74my @ssh_port_fwds;              # Queue of ssh portforwarders to start.  The
75                                # -L or -R is in here.
76my $remote_script_dir;          # location of the other sides fed-tun.pl
77my $event_repeater;             # The pathname of the event repeater
78
79if ($#ARGV != 0 && !getopts('df:rn', \%opts)) {
80    die "$usage";
81}
82
83if (defined($opts{'d'})) {
84    $debug = 1;
85}
86
87if (defined($opts{'f'})) {
88    $filename = $opts{'f'};
89}
90else {
91    # Find the configuration file in the usual place, which depends on what
92    # experiment and project we're in.
93    my $pid;
94    my $eid;
95
96    open(TMCD, "$TMCC status |") || die "tmcc failed\n";
97    print "Opened $TMCC status\n" if ($debug);
98    while (<TMCD>) {
99        chomp;
100        /ALLOCATED=([^\/]+)\/(\S+)/ && do {
101            $pid = $1;
102            $eid = $2;
103        };
104    }
105    close(TMCD);
106    my $hn = `hostname`;
107    chomp $hn;
108    $filename = "/proj/$pid/exp/$eid/tmp/$hn.gw.conf"
109        if $pid and $eid;
110}
111
112if (defined($opts{'r'})) {
113    $remote = 1;
114    die "$usage" if ($#ARGV != 1);
115   
116    $count = pop @ARGV;
117    $addr = pop @ARGV;
118}
119
120die "$usage" if (!defined($remote) && !defined($filename));
121
122if (defined($filename)) {
123    &parse_config("$filename", \%opts) || 
124        die "Cannot read config file $filename: $!\n";
125
126    foreach my $opt (@expected_opts) {
127        warn "Missing $opt option\n" if (!defined($opts{$opt}));
128    }
129
130    $active = 1 if ($opts{'active'} =~ /true/i);
131    $tunnelcfg = 1 if ($opts{'tunnelcfg'} =~ /true/i);
132    $type = $opts{'type'};
133    $type =~ tr/A-Z/a-z/;
134    $remote_script_dir = $opts{'remotescriptdir'} || ".";
135    $event_repeater = $opts{'eventrepeater'};
136    my $listen_on = $opts{'sshlisten'}||"";
137
138    if (defined($opts{'fsname'})) {
139        push(@ssh_port_fwds,"-R $listen_on:$SMBFS_PORT:$opts{'fsname'}:$SMBFS_PORT");
140    }
141
142    if (defined($opts{'bossname'})) {
143        push(@ssh_port_fwds, "-R $listen_on:$TMCD_PORT:$opts{'bossname'}:$TMCD_PORT");
144    }
145
146    if (defined($opts{'eventservername'})) {
147        push(@ssh_port_fwds,"-R ". 
148            "$listen_on:$remote_pubsub_port:$opts{'eventservername'}:$PUBSUB_PORT");
149    }
150    if (defined($opts{'remoteeventservername'})) {
151        push(@ssh_port_fwds,"-L :$remote_pubsub_port:" . 
152            "$opts{'remoteeventservername'}:$PUBSUB_PORT");
153    }
154
155    # Forward connections to seer from remote TBs to control in this TB
156    if (defined($opts{'seercontrol'})) {
157        push(@ssh_port_fwds,"-R $listen_on:$SEER_PORT:$opts{seercontrol}:$SEER_PORT");
158    }
159
160    # -n just starts the ssh tap tunnel
161    @ssh_port_fwds = () if ($opts{'type'} eq 'experiment' || $opts{'n'});
162
163    print "ssh_port_fwds = ", join("\n",@ssh_port_fwds), "\n" if ($debug);
164}
165
166# Both sides need to have GatewayPorts and PermitTunnel set.  Copy the existing
167# sshd_config, making sure GatewayPorts and PermitTunnel are set to yes,
168# replace the original, and restart sshd.
169my $ports_on = 0;
170my $tunnel_on = 0;
171
172my $conf = new IO::File($sshd_config) || die "Can't open $sshd_config: $!\n";
173my $new_conf = new IO::File(">/tmp/sshd_config") || 
174    die "Can't open new ssh_config: $!\n";
175
176while(<$conf>) {
177    s/^\s*GatewayPorts.*/GatewayPorts yes/ && do {
178        print $new_conf $_ unless $ports_on++;
179        next;
180    };
181    s/^\s*PermitTunnel.*/PermitTunnel yes/ && do {
182        print $new_conf $_ unless $tunnel_on++;
183        next;
184    };
185    print $new_conf $_;
186}
187print $new_conf "GatewayPorts yes\n" unless $ports_on;
188print $new_conf "PermitTunnel yes\n" unless $tunnel_on;
189$conf->close();
190$new_conf->close();
191
192copy("/tmp/sshd_config", $sshd_config) || 
193    die "Cannot replace $sshd_config: $!\n";
194
195system("/etc/rc.d/sshd restart");
196
197# Need these to make the Ethernet tap and bridge to work...
198system("kldload /boot/kernel/bridgestp.ko") 
199    if -r "/boot/kernel/bridgestp.ko";
200system("kldload /boot/kernel/if_bridge.ko");
201system("kldload /boot/kernel/if_tap.ko");
202
203if ($tunnelcfg && !$remote) {
204    # Most Emulab-like testbeds use globally-routable addresses on the
205    # control net;  at DETER, we isolate the control net from the Internet.
206    # On DETER-like testbeds, we need to create special tunneling nodes
207    # with external access.  Set up the external addresses as necessary.
208    &setup_tunnel_cfg(%opts);
209}
210
211if (!$remote) {
212    system("umask 077 && cp $opts{'privkeys'} /root/.ssh/id_rsa");
213    system("umask 077 && cp $opts{'pubkeys'} /root/.ssh/id_rsa.pub");
214    system("umask 077 && cat $opts{'pubkeys'} >> /root/.ssh/authorized_keys");
215}
216
217if ($active) {
218    # If we're the initiator, open up a separate tunnel to the remote
219    # host for each of the different experiment net interfaces on this
220    # machine.  Execute this startup script on the far end, but with
221    # the -r option to indicate that it's getting invoked remotely and
222    # we should just handle the remote-end tasks.
223
224    # Set up synchronization, so that the various user machines won't try to
225    # contact boss before the tunnels are set up.  (Thanks jjh!)
226
227    do {
228        system("$NC -z $opts{'peer'} $SSH_PORT");
229    } until (!$?);
230
231    # XXX:  Do we need to clear out previously created bridge interfaces?
232
233    my $count = 0;
234    my @SSHCMD;
235
236    # If we are just setting up a control net connection, just fire up
237    # ssh with the null command, and hang loose.
238
239    if ($type eq "control") {
240        foreach my $fwd (@ssh_port_fwds) {
241            system("$SSH -N $fwd -o \"StrictHostKeyChecking no\" ". 
242                "$opts{'peer'} &"); #or die "Failed to run ssh";
243        }
244
245        exit;
246    }
247
248    open(IFFILE, "/var/emulab/boot/ifmap") || die "couldn't open ifmap\n";
249    while (<IFFILE>) {
250        my @a = split(' ');
251        my $iface = $a[0];
252        my $addr = $a[1];
253        my $bridge = "bridge" . $count;
254        my $tun = "tap" . $count;
255        my $cmd;
256
257        print "Found $iface, $addr, to bridge on $bridge\n" if ($debug);
258
259        # Note that we're going to fire off an ssh which will never return;
260        # we need the connection to stay up to keep the tunnel up.
261        # In order to check for problems, we open it this way and read
262        # the expected single line of output when the tunnel is connected.
263        # To make debugging easier and to degrade more gracefully, I've split
264        # these out into multiple processes.
265       
266        foreach my $fwd (@ssh_port_fwds) {
267            $cmd = "$SSH -N $fwd -o \"StrictHostKeyChecking no\" ". 
268                "$opts{'peer'} &";
269
270            print "$cmd\n" if $debug;
271            system("$cmd"); # or die "Failed to run ssh";
272        }
273        # The Tunnel option specifies the level to tunnel at.  Ethernet creates
274        # a tap device rather than a tun device.  Strict host key checking
275        # avoids asking the user to OK a strange host key.
276        #
277        # Important safety tip:  ssh v 6.5 and above have a parsing bug
278        # that means if you put -w before the -o "Tunnel ethernet"
279        # parameter, you get a layer 3 tunnel instead of a layer 2 tunnel.
280        # Don't move args in this line capricously.
281        $cmd =  "$SSH -o \"Tunnel ethernet\" -w $count:$count " .
282            "-o \"StrictHostKeyChecking no\" " . 
283            "$opts{'peer'}  \"$remote_script_dir/fed-tun.pl " . 
284            "-r $addr $count\" & |";
285
286        print "$cmd\n" if $debug;
287
288        open(SSHCMD, $cmd) 
289           or die "Failed to run ssh";
290        # Wait for the other end to report its work done.  This string comes
291        # from the block below (after the elsif) -- tvf
292        my $check;  # Make sure something ran...
293
294        while ($check = <SSHCMD>) {
295            last if $check =~ /^Remote connection all/;
296        }
297        print "Got line [$check] from remote side\n" if ($debug);
298
299        &setup_bridging($tun, $bridge, $iface, $addr);
300        $count++;
301        @ssh_port_fwds = ();  # only do this on the first connection
302    }
303    close(IFFILE);
304
305    # Start a local event repeater (unless we're missing parameters
306    die "Missing event repeater params (No config file ?)\n"
307        unless $event_repeater && $opts{'remoteexperiment'} && 
308        $opts{'localexperiment'};
309
310    print "Starting event repeater\n" if $debug;
311   
312    print("$event_repeater -M -P $remote_pubsub_port -S localhost " . 
313        "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}\n") 
314        if $debug;
315    # Connect to the forwarded pubsub port on this host to listen to the
316    # other experiment's events, and to the local experiment to forward
317    # events.
318    system("$event_repeater -M -P $remote_pubsub_port -S localhost " . 
319        "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}");
320    warn "Event repeater returned $?\n" if $?;
321} elsif ($remote) {
322    # We're on the remote system;  figure out which interface to
323    # tweak, based on the IP address passed in.
324
325    my $iter = 0;
326    my $iface;
327
328    open(RTFILE, "$ROUTE_GET $addr |") || die "couldn't do $ROUTE_GET\n";
329    while (<RTFILE>) {
330        if (/interface: (\w+)/) {
331            $iface = $1;
332        }
333    }
334    close(RTFILE);
335
336    die "Couldn't find interface to use." if (!defined($iface));
337    my $bridge = "bridge" . $count;
338    my $tun = "tap" . $count;
339
340    &setup_bridging($tun, $bridge, $iface, $addr);
341    print "Remote connection all set up!\n";  # Trigger other end with output
342
343    # If this is the first remote invocation on a control gateway, start the
344    # event repeater.
345
346    if ( $count == 0 && $type ne "experiment" ) {
347        my $remote_pubsub_port = $PUBSUB_PORT - 1;  # There will be a local
348                                                    # pubsubd running, so we
349                                                    # dodge the port on the
350                                                    # remote tunnel node.
351        # Make sure we have the relevant parameters
352        die "Missing event repeater params (No config file ?)\n"
353            unless $event_repeater && $opts{'remoteexperiment'} && 
354            $opts{'localexperiment'};
355
356        print "Starting event repeater\n" if $debug;
357       
358        # Connect to the forwarded pubsub port on this host to listen to the
359        # other experiment's events, and to the local experiment to forward
360        # events.
361        system("$event_repeater -P $remote_pubsub_port -S localhost " . 
362            "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}");
363        warn "Event repeater returned $?\n" if $?;
364    }
365} else {
366    print "inactive end of a connection, finishing" if ($debug);
367}
368
369print "all done!\n" if ($debug);
370exit;
371
372
373# Set up the bridging for the new stuff...
374
375sub setup_bridging($; $; $; $) {
376    my ($tun, $bridge, $iface, $addr) = @_;
377
378    print "Waiting to see if new iface $tun is up\n" if ($debug);
379
380    do {
381        sleep 1;
382        system("$IFCONFIG $tun");
383    } until (!$?);
384
385    print "setting up $bridge with $iface and nuking $addr\n" if ($debug);
386
387    system("ifconfig $bridge create");
388    system("ifconfig $iface delete $addr");
389    system("ifconfig $bridge addm $iface up");
390    system("ifconfig $bridge addm $tun");
391}
392
393# Set up tunnel info for DETER-like testbeds.
394
395sub setup_tunnel_cfg {
396    my (%opts) = @_;
397    #my $tunnel_iface = "em0";   # XXX
398    my $tunnel_iface = $opts{'interface'} || "em0";
399    my $tunnel_ip;
400    my $tunnel_mask;
401    my $tunnel_mac;
402    my $tunnel_router;
403
404    print "Opening $TMCC tunnelip\n" if ($debug);
405
406    open(TMCD, "$TMCC tunnelip |") || die "tmcc failed\n";
407    print "Opened $TMCC tunnelip\n" if ($debug);
408    while (<TMCD>) {
409        print "got one line from tmcc\n" if ($debug);
410        print if ($debug);
411        if (/^TUNNELIP=([0-9.]*) TUNNELMASK=([0-9.]*) TUNNELMAC=(\w*) TUNNELROUTER=([0-9.]*)$/) {
412            $tunnel_ip = $1;
413            $tunnel_mask = $2;
414            $tunnel_mac = $3;
415            $tunnel_router = $4;
416        }
417    }
418    close(TMCD);
419
420    print "tunnel options:  ip=$tunnel_ip mask=$tunnel_mask mac=$tunnel_mac router=$tunnel_router\n" if ($debug);
421
422    if (defined($tunnel_mac) && ($tunnel_mac ne "")) {
423       print "Opening $FINDIF $tunnel_mac\n" if ($debug);
424       open(FIF, "$FINDIF $tunnel_mac |") || die "findif failed\n";
425       my $line = <FIF>;
426       chomp($line);
427       print "$FINDIF returned $line\n" if ($debug);
428       $tunnel_iface = $line;
429    }
430
431    system("ifconfig $tunnel_iface $tunnel_ip" .
432           ($tunnel_mask ? " netmask $tunnel_mask" : "") . " up");
433    warn "configuration of tunnel interface failed" if ($?);
434
435    if ($tunnel_router) {
436
437        # Sometimes the insertion of DNS names lags a bit.  Retry this
438        # configuration a few times to let DNS catch up.  Might want to really
439        # check the DNS name before we try this...
440        my $config_succeeded = 0;
441        my $tries = 0;
442        my $max_retries = 300;
443
444        do {
445            system("route add $opts{'peer'} $tunnel_router");
446            if ( $? ) {
447                warn "configuration routes via tunnel interface failed";
448                $tries++;
449                sleep(10);
450            }
451            else { $config_succeeded = 1; }
452        } until ( $config_succeeded || $tries > $max_retries );
453    }
454
455    print "setup_tunnel_cfg done\n" if ($debug);
456}
457
458# Trick config-file parsing code from Ted Faber:
459
460# Parse the config file.  The format is a colon-separated parameter name
461# followed by the value of that parameter to the end of the line.  This parses
462# that format and puts the parameters into the referenced hash.  Parameter
463# names are mapped to lower case, parameter values are unchanged.  Returns 0 on
464# failure (e.g. file open) and 1 on success.
465sub parse_config {
466    my($file, $href) = @_;
467    my($fh) = new IO::File($file);
468       
469    unless ($fh) {
470        warn "Can't open $file: $!\n";
471        return 0;
472    }
473
474    while (<$fh>) {
475        next if /^\s*#/ || /^\s*$/;     # Skip comments & blanks
476        chomp;
477        /^([^:]+):\s*(.*)/ && do {
478            my($key) = $1; 
479
480            $key =~ tr/A-Z/a-z/;
481            $href->{$key} = $2;
482            next;
483        };
484        warn "Unparasble line in $file: $_\n";
485    }
486    $fh->close();   # It will close when it goes out of scope, but...
487    return 1;
488}
Note: See TracBrowser for help on using the repository browser.