source: fedkit/fed-tun.ucb.pl @ 6679c122

axis_examplecompt_changesinfo-opsversion-1.30version-2.00version-3.01version-3.02
Last change on this file since 6679c122 was 2aeb39e, checked in by Ted Faber <faber@…>, 17 years ago

demo done

  • Property mode set to 100755
File size: 11.4 KB
Line 
1#!/usr/bin/perl -w
2
3# Kevin Lahey, lahey@isi.edu
4# July 11, 2007
5
6# Set up ssh tunnel infrastructure for federation:
7#
8# * Parse the configuration file provided.
9#
10# * Figure out whether we're the initiator or the reciever;  if we're
11#   the receiver, we just need to set up ssh keys and exit.
12#
13# * Pick out the experimental interface, remove the IP address
14#
15# * Create a layer 2 ssh tunnel, set up bridging, and hang loose.
16#
17
18use strict;
19use Getopt::Std;
20use POSIX qw(strftime);
21use Sys::Hostname;
22use IO::File;
23use IO::Pipe;
24
25my $IFCONFIG = "/sbin/ifconfig";
26my $TMCC = "/usr/local/etc/emulab/tmcc";
27my $SSH = "/usr/local/bin/ssh";
28my $NC = "/usr/bin/nc";
29my $SSH_PORT = 22;
30my $ROUTE_GET = "/sbin/route get";  # XXX:  works on FreeBSD, but should
31                                    #       should be 'ip route get' for Linux
32
33# Ports that are forwarded between testbeds
34my $TMCD_PORT = 7777;
35my $SMBFS_PORT = 139;
36my $PUBSUB_PORT = 16505;
37
38
39my $remote_pubsub_port = $PUBSUB_PORT - 1;  # There will be a local
40                                            # pubsubd running, so we
41                                            # dodge the port on the
42                                            # remote tunnel node.
43sub setup_bridging;
44sub setup_tunnel_cfg;
45sub parse_config;
46
47# Option use is as follows:
48#    -f         filename containing config file
49#    -d         turn on debugging output
50#    -r         remotely invoked
51#               (if remotely invoked the last two args are the tun interface
52#               number and the remote address of the tunnel)
53
54my $usage = "Usage: fed-tun.pl [-r] [-d] [-f config-filename] [count addr]\n";
55
56my %opts;       # Note that this is used for both getops and the options file
57my @expected_opts = qw(active tunnelcfg bossname fsname type
58                       peer pubkeys privkeys);
59my $filename;
60my $remote;
61my $debug = 1;
62my $count;
63my $addr;
64my $active;
65my $type;
66my $tunnelcfg;
67my $ssh_port_fwds = "";
68my $remote_script_dir;          # location of the other sides fed-tun.pl
69my $event_repeater;             # The pathname of the event repeater
70my $remote_config_file;         # Config file for the other side
71
72if ($#ARGV != 0 && !getopts('df:r', \%opts)) {
73    die "$usage";
74}
75
76if (defined($opts{'d'})) {
77    $debug = 1;
78}
79
80if (defined($opts{'f'})) {
81    $filename = $opts{'f'};
82}
83
84if (defined($opts{'r'})) {
85    $remote = 1;
86    die "$usage" if ($#ARGV != 1);
87   
88    $count = pop @ARGV;
89    $addr = pop @ARGV;
90}
91
92die "$usage" if (!defined($remote) && !defined($filename));
93
94if (defined($filename)) {
95    &parse_config("$filename", \%opts) || 
96        die "Cannot read config file $filename: $!\n";
97
98    foreach my $opt (@expected_opts) {
99        warn "Missing $opt option\n" if (!defined($opts{$opt}));
100    }
101
102    $active = 1 if ($opts{'active'} =~ /true/i);
103    $tunnelcfg = 1 if ($opts{'tunnelcfg'} =~ /true/i);
104    $type = $opts{'type'};
105    $type =~ tr/A-Z/a-z/;
106    $remote_script_dir = $opts{'remotescriptdir'} || ".";
107    $event_repeater = $opts{'eventrepeater'};
108    $remote_config_file = $opts{'remoteconfigfile'};
109    $remote_config_file = "-f $remote_config_file" if $remote_config_file;
110
111    if (defined($opts{'fsname'})) {
112        $ssh_port_fwds = "-R :$SMBFS_PORT:$opts{'fsname'}:$SMBFS_PORT ";
113    }
114
115    if (defined($opts{'bossname'})) {
116        $ssh_port_fwds .= "-R :$TMCD_PORT:$opts{'bossname'}:$TMCD_PORT ";
117    }
118
119    if (defined($opts{'eventservername'})) {
120        $ssh_port_fwds .= "-R :$remote_pubsub_port:$opts{'eventservername'}:" . 
121            "$PUBSUB_PORT ";
122    }
123    if (defined($opts{'remoteeventservername'})) {
124        $ssh_port_fwds .= "-L :$remote_pubsub_port:" . 
125            "$opts{'remoteeventservername'}:$PUBSUB_PORT ";
126    }
127
128    $ssh_port_fwds = "" if ($opts{'type'} eq 'experiment');
129
130    print "ssh_port_fwds = $ssh_port_fwds\n" if ($debug);
131}
132
133# Need these to make the Ethernet tap and bridge to work...
134
135system("kldload /boot/kernel/if_bridge.ko");
136system("kldload /boot/kernel/if_tap.ko");
137
138if (!$remote) {
139    # At the Berkeley federant, the outgoing addresses are fixed and based on
140    # the control net.  We do set up a host route to the peer.  This is a
141    # simplified version of the DETER setup.
142
143    &setup_tunnel_cfg(%opts);
144}
145
146if (!$remote) {
147    system("umask 077 && cp $opts{'privkeys'} /root/.ssh/id_rsa");
148    system("umask 077 && cp $opts{'pubkeys'} /root/.ssh/id_rsa.pub");
149    system("umask 077 && cat $opts{'pubkeys'} >> /root/.ssh/authorized_keys");
150}
151
152if ($active) {
153    # If we're the initiator, open up a separate tunnel to the remote
154    # host for each of the different experiment net interfaces on this
155    # machine.  Execute this startup script on the far end, but with
156    # the -r option to indicate that it's getting invoked remotely and
157    # we should just handle the remote-end tasks.
158
159    # Set up synchronization, so that the various user machines won't try to
160    # contact boss before the tunnels are set up.  (Thanks jjh!)
161
162    do {
163        system("$NC -z $opts{'peer'} $SSH_PORT");
164    } until (!$?);
165
166    # XXX:  Do we need to clear out previously created bridge interfaces?
167
168    my $count = 0;
169    my @SSHCMD;
170
171    # If we are just setting up a control net connection, just fire up
172    # ssh with the null command, and hang loose.
173
174    if ($type eq "control") {
175        system("$SSH $ssh_port_fwds -Nno \"StrictHostKeyChecking no\" $opts{'peer'} &"); #or die "Failed to run ssh";
176
177        exit;
178    }
179
180    open(IFFILE, "/var/emulab/boot/ifmap") || die "couldn't open ifmap\n";
181    while (<IFFILE>) {
182        my @a = split(' ');
183        my $iface = $a[0];
184        my $addr = $a[1];
185        my $bridge = "bridge" . $count;
186        my $tun = "tap" . $count;
187
188        print "Found $iface, $addr, to bridge on $bridge\n" if ($debug);
189
190        # Note that we're going to fire off an ssh which will never return;
191        # we need the connection to stay up to keep the tunnel up.
192        # In order to check for problems, we open it this way and read
193        # the expected single line of output when the tunnel is connected.
194
195        print "$SSH -w $count:$count $ssh_port_fwds -o \"StrictHostKeyChecking no\" $opts{'peer'}  \"$remote_script_dir/fed-tun.pl $remote_config_file -r $addr $count\"\n" if $debug;
196        open($SSHCMD[$count], "$SSH -w $count:$count $ssh_port_fwds -o \"StrictHostKeyChecking no\" $opts{'peer'}  \"$remote_script_dir/fed-tun.pl $remote_config_file -r $addr $count\" |") or die "Failed to run ssh";
197
198        my $check = <$SSHCMD[$count]>;  # Make sure something ran...
199        print "Got line [$check] from remote side\n" if ($debug);
200
201        &setup_bridging($tun, $bridge, $iface, $addr);
202        $count++;
203        $ssh_port_fwds = "";  # only do this on the first connection
204    }
205    close(IFFILE);
206
207    # Start a local event repeater (unless we're missing parameters
208    die "Missing event repeater params (No config file ?)\n"
209        unless $event_repeater && $opts{'remoteexperiment'} && 
210        $opts{'localexperiment'};
211
212    print "Starting event repeater\n" if $debug;
213   
214    print("$event_repeater -M -P $remote_pubsub_port -S localhost " . 
215        "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}\n") 
216        if $debug;
217    # Connect to the forwarded pubsub port on this host to listen to the
218    # other experiment's events, and to the local experiment to forward
219    # events.
220    system("$event_repeater -M -P $remote_pubsub_port -S localhost " . 
221        "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}");
222    warn "Event repeater returned $?\n" if $?;
223} elsif ($remote) {
224    # We're on the remote system;  figure out which interface to
225    # tweak, based on the IP address passed in.
226
227    my $iter = 0;
228    my $iface;
229
230    open(RTFILE, "$ROUTE_GET $addr |") || die "couldn't do $ROUTE_GET\n";
231    while (<RTFILE>) {
232        if (/interface: (\w+)/) {
233            $iface = $1;
234        }
235    }
236    close(RTFILE);
237
238    die "Couldn't find interface to use." if (!defined($iface));
239    my $bridge = "bridge" . $count;
240    my $tun = "tap" . $count;
241
242    &setup_bridging($tun, $bridge, $iface, $addr);
243    print "Remote connection all set up!\n";  # Trigger other end with output
244
245    # If this is the first remote invocation on a control gateway, start the
246    # event repeater.
247
248    my $file = new IO::File(">/tmp/remote");
249    print($file "hello!!!\n") if $file;
250    if ( $count == 0 && $type ne "experiment" ) {
251        my $remote_pubsub_port = $PUBSUB_PORT - 1;  # There will be a local
252                                                    # pubsubd running, so we
253                                                    # dodge the port on the
254                                                    # remote tunnel node.
255        print($file "In here!\n");
256        # Make sure we have the relevant parameters
257        die "Missing event repeater params (No config file ?)\n"
258            unless $event_repeater && $opts{'remoteexperiment'} && 
259            $opts{'localexperiment'};
260
261        print "Starting event repeater\n" if $debug;
262       
263        print($file "$event_repeater -P $remote_pubsub_port -S localhost " . 
264            "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}\n") 
265            if $file;
266        # Connect to the forwarded pubsub port on this host to listen to the
267        # other experiment's events, and to the local experiment to forward
268        # events.
269        system("$event_repeater -P $remote_pubsub_port -S localhost " . 
270            "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}");
271        warn "Event repeater returned $?\n" if $?;
272    }
273    $file->close() if $file;
274} else {
275    print "inactive end of a connection, finishing" if ($debug);
276}
277
278print "all done!\n" if ($debug);
279exit;
280
281
282# Set up the bridging for the new stuff...
283
284sub setup_bridging($; $; $; $) {
285    my ($tun, $bridge, $iface, $addr) = @_;
286
287    print "Waiting to see if new iface $tun is up\n" if ($debug);
288
289    do {
290        sleep 1;
291        system("$IFCONFIG $tun");
292    } until (!$?);
293
294    print "setting up $bridge with $iface and nuking $addr\n" if ($debug);
295
296    system("ifconfig $bridge create");
297    system("ifconfig $iface delete $addr");
298    system("ifconfig $bridge addm $iface up");
299    system("ifconfig $bridge addm $tun");
300}
301
302# Set up tunnel info for UCB testbed
303
304sub setup_tunnel_cfg {
305    my (%opts) = @_;
306    my $tunnel_iface = "em5";
307    my $control_iface = "em4";
308    my $tunnel_prefix = "192.154.6.";
309    my $tunnel_router = "192.154.6.1";
310    my $netmask = "255.255.255.0";
311    my $ipipe = new IO::Pipe();
312    my $pcnum;
313
314    $ipipe->reader("$IFCONFIG $control_iface");
315
316    while (<$ipipe>) {
317        /inet\s+\d+\.\d+\.\d+\.(\d+)/ && do {
318            $pcnum = $1;
319            last;
320        };
321    }
322    $ipipe->close();
323    $pcnum += 50;
324    $pcnum &=0xff if $pcnum > 255;
325
326    system("ifconfig $tunnel_iface $tunnel_prefix$pcnum netmask $netmask");
327    warn "configuration of tunnel interface failed" if ($?);
328
329    # Sometimes the insertion of DNS names lags a bit.  Retry this
330    # configuration a few times to let DNS catch up.  Might want to really
331    # check the DNS name before we try this...
332    my $config_succeeded = 0;
333    my $tries = 0;
334    my $max_retries = 30;
335
336    do {
337        system("route add $opts{'peer'} $tunnel_router");
338        if ( $? ) {
339            warn "configuration routes via tunnel interface failed";
340            $tries++;
341            sleep(10);
342        }
343        else { $config_succeeded = 1; }
344    } until ( $config_succeeded || $tries > $max_retries );
345
346    print "setup_tunnel_cfg done\n" if ($debug);
347}
348
349# Trick config-file parsing code from Ted Faber:
350
351# Parse the config file.  The format is a colon-separated parameter name
352# followed by the value of that parameter to the end of the line.  This parses
353# that format and puts the parameters into the referenced hash.  Parameter
354# names are mapped to lower case, parameter values are unchanged.  Returns 0 on
355# failure (e.g. file open) and 1 on success.
356sub parse_config {
357    my($file, $href) = @_;
358    my($fh) = new IO::File($file);
359       
360    unless ($fh) {
361        warn "Can't open $file: $!\n";
362        return 0;
363    }
364
365    while (<$fh>) {
366        next if /^\s*#/ || /^\s*$/;     # Skip comments & blanks
367        chomp;
368        /^([^:]+):\s*(.*)/ && do {
369            my($key) = $1; 
370
371            $key =~ tr/A-Z/a-z/;
372            $href->{$key} = $2;
373            next;
374        };
375        warn "Unparasble line in $file: $_\n";
376    }
377    $fh->close();   # It will close when it goes out of scope, but...
378    return 1;
379}
Note: See TracBrowser for help on using the repository browser.