source: fedkit/fed-tun.pl @ 7a8d667

axis_examplecompt_changesinfo-opsversion-1.30version-2.00version-3.01version-3.02
Last change on this file since 7a8d667 was 7a8d667, checked in by Ted Faber <faber@…>, 16 years ago

reconfigure sshd_config explicitly

  • Property mode set to 100755
File size: 13.2 KB
Line 
1#!/usr/bin/perl -w
2
3# Kevin Lahey, lahey@isi.edu
4# July 11, 2007
5
6# Set up ssh tunnel infrastructure for federation:
7#
8# * Parse the configuration file provided.
9#
10# * Figure out whether we're the initiator or the reciever;  if we're
11#   the receiver, we just need to set up ssh keys and exit.
12#
13# * Pick out the experimental interface, remove the IP address
14#
15# * Create a layer 2 ssh tunnel, set up bridging, and hang loose.
16#
17
18use strict;
19use Getopt::Std;
20use POSIX qw(strftime);
21use Sys::Hostname;
22use IO::File;
23use File::Copy;
24
25my $IFCONFIG = "/sbin/ifconfig";
26my $TMCC = "/usr/local/etc/emulab/tmcc";
27my $SSH = "/usr/local/bin/ssh";
28my $NC = "/usr/bin/nc";
29my $SSH_PORT = 22;
30my $ROUTE_GET = "/sbin/route get";  # XXX:  works on FreeBSD, but should
31                                    #       should be 'ip route get' for Linux
32my $sshd_config = "/etc/ssh/sshd_config";    # Probably should be a param
33
34# Ports that are forwarded between testbeds
35my $TMCD_PORT = 7777;
36my $SMBFS_PORT = 139;
37my $PUBSUB_PORT = 16505;
38my $SEER_PORT = 16606;
39
40
41my $remote_pubsub_port = $PUBSUB_PORT - 1;  # There will be a local
42                                            # pubsubd running, so we
43                                            # dodge the port on the
44                                            # remote tunnel node.
45sub setup_bridging;
46sub setup_tunnel_cfg;
47sub parse_config;
48
49# Option use is as follows:
50#    -f         filename containing config file
51#    -d         turn on debugging output
52#    -r         remotely invoked
53#               (if remotely invoked the last two args are the tun interface
54#               number and the remote address of the tunnel)
55
56my $usage = "Usage: fed-tun.pl [-r] [-d] [-f config-filename] [count addr]\n";
57
58my %opts;       # Note that this is used for both getops and the options file
59my @expected_opts = qw(active tunnelcfg bossname fsname type
60                       peer pubkeys privkeys);
61my $filename;
62my $remote;
63my $debug = 1;
64my $count;
65my $addr;
66my $active;
67my $type;
68my $tunnelcfg;
69my @ssh_port_fwds;              # Queue of ssh portforwarders to start.  The
70                                # -L or -R is in here.
71my $remote_script_dir;          # location of the other sides fed-tun.pl
72my $event_repeater;             # The pathname of the event repeater
73my $remote_config_file;         # Config file for the other side
74
75if ($#ARGV != 0 && !getopts('df:rn', \%opts)) {
76    die "$usage";
77}
78
79if (defined($opts{'d'})) {
80    $debug = 1;
81}
82
83if (defined($opts{'f'})) {
84    $filename = $opts{'f'};
85}
86
87if (defined($opts{'r'})) {
88    $remote = 1;
89    die "$usage" if ($#ARGV != 1);
90   
91    $count = pop @ARGV;
92    $addr = pop @ARGV;
93}
94
95die "$usage" if (!defined($remote) && !defined($filename));
96
97if (defined($filename)) {
98    &parse_config("$filename", \%opts) || 
99        die "Cannot read config file $filename: $!\n";
100
101    foreach my $opt (@expected_opts) {
102        warn "Missing $opt option\n" if (!defined($opts{$opt}));
103    }
104
105    $active = 1 if ($opts{'active'} =~ /true/i);
106    $tunnelcfg = 1 if ($opts{'tunnelcfg'} =~ /true/i);
107    $type = $opts{'type'};
108    $type =~ tr/A-Z/a-z/;
109    $remote_script_dir = $opts{'remotescriptdir'} || ".";
110    $event_repeater = $opts{'eventrepeater'};
111    $remote_config_file = $opts{'remoteconfigfile'};
112    $remote_config_file = "-f $remote_config_file" if $remote_config_file;
113
114    if (defined($opts{'fsname'})) {
115        push(@ssh_port_fwds,"-R :$SMBFS_PORT:$opts{'fsname'}:$SMBFS_PORT");
116    }
117
118    if (defined($opts{'bossname'})) {
119        push(@ssh_port_fwds, "-R :$TMCD_PORT:$opts{'bossname'}:$TMCD_PORT");
120    }
121
122    if (defined($opts{'eventservername'})) {
123        push(@ssh_port_fwds,"-R ". 
124            ":$remote_pubsub_port:$opts{'eventservername'}:$PUBSUB_PORT");
125    }
126    if (defined($opts{'remoteeventservername'})) {
127        push(@ssh_port_fwds,"-L :$remote_pubsub_port:" . 
128            "$opts{'remoteeventservername'}:$PUBSUB_PORT");
129    }
130
131    # Forward connections to seer from remote TBs to control in this TB
132    if (defined($opts{'seercontrol'})) {
133        push(@ssh_port_fwds,"-R :$SEER_PORT:$opts{seercontrol}:$SEER_PORT");
134    }
135
136    # -n just starts the ssh tap tunnel
137    @ssh_port_fwds = () if ($opts{'type'} eq 'experiment' || $opts{'n'});
138
139    print "ssh_port_fwds = ", join("\n",@ssh_port_fwds), "\n" if ($debug);
140}
141
142# Both sides need to have GatewayPorts to be set.  Copy the existing
143# sshd_config, making sure GatewayPorts is set to yes, replace the original,
144# and restart sshd.
145my $ports_on = 0;
146
147my $conf = new IO::File($sshd_config) || die "Can't open $sshd_config: $!\n";
148my $new_conf = new IO::File(">/tmp/sshd_config") || 
149    die "Can't open new ssh_config: $!\n";
150
151while(<$conf>) {
152    s/^\s*GatewayPorts.*/GatewayPorts yes/ && do {
153        print $new_conf $_ unless $ports_on++;
154        next;
155    };
156    print $new_conf $_;
157}
158print $new_conf "GatewayPorts yes\n" unless $ports_on;
159$conf->close();
160$new_conf->close();
161
162copy("/tmp/sshd_config", $sshd_config) || 
163    die "Cannot replace $sshd_config: $!\n";
164
165system("/etc/rc.d/sshd restart");
166
167# Need these to make the Ethernet tap and bridge to work...
168
169system("kldload /boot/kernel/if_bridge.ko");
170system("kldload /boot/kernel/if_tap.ko");
171
172if ($tunnelcfg && !$remote) {
173    # Most Emulab-like testbeds use globally-routable addresses on the
174    # control net;  at DETER, we isolate the control net from the Internet.
175    # On DETER-like testbeds, we need to create special tunneling nodes
176    # with external access.  Set up the external addresses as necessary.
177    &setup_tunnel_cfg(%opts);
178}
179
180if (!$remote) {
181    system("umask 077 && cp $opts{'privkeys'} /root/.ssh/id_rsa");
182    system("umask 077 && cp $opts{'pubkeys'} /root/.ssh/id_rsa.pub");
183    system("umask 077 && cat $opts{'pubkeys'} >> /root/.ssh/authorized_keys");
184}
185
186if ($active) {
187    # If we're the initiator, open up a separate tunnel to the remote
188    # host for each of the different experiment net interfaces on this
189    # machine.  Execute this startup script on the far end, but with
190    # the -r option to indicate that it's getting invoked remotely and
191    # we should just handle the remote-end tasks.
192
193    # Set up synchronization, so that the various user machines won't try to
194    # contact boss before the tunnels are set up.  (Thanks jjh!)
195
196    do {
197        system("$NC -z $opts{'peer'} $SSH_PORT");
198    } until (!$?);
199
200    # XXX:  Do we need to clear out previously created bridge interfaces?
201
202    my $count = 0;
203    my @SSHCMD;
204
205    # If we are just setting up a control net connection, just fire up
206    # ssh with the null command, and hang loose.
207
208    if ($type eq "control") {
209        foreach my $fwd (@ssh_port_fwds) {
210            system("$SSH -N $fwd -Nno \"StrictHostKeyChecking no\" ". 
211                "$opts{'peer'} &"); #or die "Failed to run ssh";
212        }
213
214        exit;
215    }
216
217    open(IFFILE, "/var/emulab/boot/ifmap") || die "couldn't open ifmap\n";
218    while (<IFFILE>) {
219        my @a = split(' ');
220        my $iface = $a[0];
221        my $addr = $a[1];
222        my $bridge = "bridge" . $count;
223        my $tun = "tap" . $count;
224        my $cmd;
225
226        print "Found $iface, $addr, to bridge on $bridge\n" if ($debug);
227
228        # Note that we're going to fire off an ssh which will never return;
229        # we need the connection to stay up to keep the tunnel up.
230        # In order to check for problems, we open it this way and read
231        # the expected single line of output when the tunnel is connected.
232        # To make debugging easier and to degrade more gracefully, I've split
233        # these out into multiple processes.
234       
235        foreach my $fwd (@ssh_port_fwds) {
236            $cmd = "$SSH -N $fwd -o \"StrictHostKeyChecking no\" ". 
237                "$opts{'peer'} &";
238
239            print "$cmd\n" if $debug;
240            system("$cmd"); # or die "Failed to run ssh";
241        }
242        $cmd =  "$SSH -w $count:$count -o \"StrictHostKeyChecking no\" " . 
243            "$opts{'peer'}  \"$remote_script_dir/fed-tun.pl " . 
244            "$remote_config_file -r $addr $count\" & |";
245
246        print "$cmd\n" if $debug;
247
248        open($SSHCMD[$count], $cmd) 
249           or die "Failed to run ssh";
250
251        my $check = <$SSHCMD[$count]>;  # Make sure something ran...
252        print "Got line [$check] from remote side\n" if ($debug);
253
254        &setup_bridging($tun, $bridge, $iface, $addr);
255        $count++;
256        @ssh_port_fwds = ();  # only do this on the first connection
257    }
258    close(IFFILE);
259
260    # Start a local event repeater (unless we're missing parameters
261    die "Missing event repeater params (No config file ?)\n"
262        unless $event_repeater && $opts{'remoteexperiment'} && 
263        $opts{'localexperiment'};
264
265    print "Starting event repeater\n" if $debug;
266   
267    print("$event_repeater -M -P $remote_pubsub_port -S localhost " . 
268        "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}\n") 
269        if $debug;
270    # Connect to the forwarded pubsub port on this host to listen to the
271    # other experiment's events, and to the local experiment to forward
272    # events.
273    system("$event_repeater -M -P $remote_pubsub_port -S localhost " . 
274        "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}");
275    warn "Event repeater returned $?\n" if $?;
276} elsif ($remote) {
277    # We're on the remote system;  figure out which interface to
278    # tweak, based on the IP address passed in.
279
280    my $iter = 0;
281    my $iface;
282
283    open(RTFILE, "$ROUTE_GET $addr |") || die "couldn't do $ROUTE_GET\n";
284    while (<RTFILE>) {
285        if (/interface: (\w+)/) {
286            $iface = $1;
287        }
288    }
289    close(RTFILE);
290
291    die "Couldn't find interface to use." if (!defined($iface));
292    my $bridge = "bridge" . $count;
293    my $tun = "tap" . $count;
294
295    &setup_bridging($tun, $bridge, $iface, $addr);
296    print "Remote connection all set up!\n";  # Trigger other end with output
297
298    # If this is the first remote invocation on a control gateway, start the
299    # event repeater.
300
301    if ( $count == 0 && $type ne "experiment" ) {
302        my $remote_pubsub_port = $PUBSUB_PORT - 1;  # There will be a local
303                                                    # pubsubd running, so we
304                                                    # dodge the port on the
305                                                    # remote tunnel node.
306        # Make sure we have the relevant parameters
307        die "Missing event repeater params (No config file ?)\n"
308            unless $event_repeater && $opts{'remoteexperiment'} && 
309            $opts{'localexperiment'};
310
311        print "Starting event repeater\n" if $debug;
312       
313        # Connect to the forwarded pubsub port on this host to listen to the
314        # other experiment's events, and to the local experiment to forward
315        # events.
316        system("$event_repeater -P $remote_pubsub_port -S localhost " . 
317            "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}");
318        warn "Event repeater returned $?\n" if $?;
319    }
320} else {
321    print "inactive end of a connection, finishing" if ($debug);
322}
323
324print "all done!\n" if ($debug);
325exit;
326
327
328# Set up the bridging for the new stuff...
329
330sub setup_bridging($; $; $; $) {
331    my ($tun, $bridge, $iface, $addr) = @_;
332
333    print "Waiting to see if new iface $tun is up\n" if ($debug);
334
335    do {
336        sleep 1;
337        system("$IFCONFIG $tun");
338    } until (!$?);
339
340    print "setting up $bridge with $iface and nuking $addr\n" if ($debug);
341
342    system("ifconfig $bridge create");
343    system("ifconfig $iface delete $addr");
344    system("ifconfig $bridge addm $iface up");
345    system("ifconfig $bridge addm $tun");
346}
347
348# Set up tunnel info for DETER-like testbeds.
349
350sub setup_tunnel_cfg {
351    my (%opts) = @_;
352    my $tunnel_iface = "em0";   # XXX
353    my $tunnel_ip;
354    my $tunnel_mask;
355    my $tunnel_mac;
356    my $tunnel_router;
357
358    print "Opening $TMCC tunnelip\n" if ($debug);
359
360    open(TMCD, "$TMCC tunnelip |") || die "tmcc failed\n";
361    print "Opened $TMCC tunnelip\n" if ($debug);
362    while (<TMCD>) {
363        print "got one line from tmcc\n" if ($debug);
364        print if ($debug);
365        if (/^TUNNELIP=([0-9.]*) TUNNELMASK=([0-9.]*) TUNNELMAC=(\w*) TUNNELROUTER=([0-9.]*)$/) {
366            $tunnel_ip = $1;
367            $tunnel_mask = $2;
368            $tunnel_mac = $3;
369            $tunnel_router = $4;
370        }
371    }
372    close(TMCD);
373
374    die "Unable to determine tunnel node configuration information"
375        if (!defined($tunnel_router));
376
377    print "tunnel options:  ip=$tunnel_ip mask=$tunnel_mask mac=$tunnel_mac router=$tunnel_router\n" if ($debug);
378
379    # Sadly, we ignore the tunnel mac for now -- we should eventually
380    # use it to determine which interface to use, just like the
381    # Emulab startup scripts.
382
383    system("ifconfig $tunnel_iface $tunnel_ip" .
384           ($tunnel_mask ? " netmask $tunnel_mask" : "") . " up");
385    warn "configuration of tunnel interface failed" if ($?);
386
387    # Sometimes the insertion of DNS names lags a bit.  Retry this
388    # configuration a few times to let DNS catch up.  Might want to really
389    # check the DNS name before we try this...
390    my $config_succeeded = 0;
391    my $tries = 0;
392    my $max_retries = 30;
393
394    do {
395        system("route add $opts{'peer'} $tunnel_router");
396        if ( $? ) {
397            warn "configuration routes via tunnel interface failed";
398            $tries++;
399            sleep(10);
400        }
401        else { $config_succeeded = 1; }
402    } until ( $config_succeeded || $tries > $max_retries );
403
404    print "setup_tunnel_cfg done\n" if ($debug);
405}
406
407# Trick config-file parsing code from Ted Faber:
408
409# Parse the config file.  The format is a colon-separated parameter name
410# followed by the value of that parameter to the end of the line.  This parses
411# that format and puts the parameters into the referenced hash.  Parameter
412# names are mapped to lower case, parameter values are unchanged.  Returns 0 on
413# failure (e.g. file open) and 1 on success.
414sub parse_config {
415    my($file, $href) = @_;
416    my($fh) = new IO::File($file);
417       
418    unless ($fh) {
419        warn "Can't open $file: $!\n";
420        return 0;
421    }
422
423    while (<$fh>) {
424        next if /^\s*#/ || /^\s*$/;     # Skip comments & blanks
425        chomp;
426        /^([^:]+):\s*(.*)/ && do {
427            my($key) = $1; 
428
429            $key =~ tr/A-Z/a-z/;
430            $href->{$key} = $2;
431            next;
432        };
433        warn "Unparasble line in $file: $_\n";
434    }
435    $fh->close();   # It will close when it goes out of scope, but...
436    return 1;
437}
Note: See TracBrowser for help on using the repository browser.