#!/usr/bin/perl -w # Kevin Lahey, lahey@isi.edu # July 11, 2007 # Set up ssh tunnel infrastructure for federation: # # * Parse the configuration file provided. # # * Figure out whether we're the initiator or the reciever; if we're # the receiver, we just need to set up ssh keys and exit. # # * Pick out the experimental interface, remove the IP address # # * Create a layer 2 ssh tunnel, set up bridging, and hang loose. # use strict; use Getopt::Std; use POSIX qw(strftime); use Sys::Hostname; use IO::File; use IO::Pipe; use File::Copy; my $IFCONFIG = "/sbin/ifconfig"; my $TMCC = "/usr/local/etc/emulab/tmcc"; # If a special version of ssh is required, it will be installed in # /usr/local/bin/ssh. Otherwise use the usual one. my $SSH = -x "/usr/local/bin/ssh" ? "/usr/local/bin/ssh" : "/usr/bin/ssh"; my $NC = "/usr/bin/nc"; my $SSH_PORT = 22; my $ROUTE_GET = "/sbin/route get"; # XXX: works on FreeBSD, but should # should be 'ip route get' for Linux my $sshd_config = "/etc/ssh/sshd_config"; # Probably should be a param # Ports that are forwarded between testbeds my $TMCD_PORT = 7777; my $SMBFS_PORT = 139; my $PUBSUB_PORT = 16505; my $remote_pubsub_port = $PUBSUB_PORT - 1; # There will be a local # pubsubd running, so we # dodge the port on the # remote tunnel node. die "Cannot exec $SSH" unless -x $SSH; sub setup_bridging; sub setup_tunnel_cfg; sub parse_config; # Option use is as follows: # -f filename containing config file # -d turn on debugging output # -r remotely invoked # (if remotely invoked the last two args are the tun interface # number and the remote address of the tunnel) my $usage = "Usage: fed-tun.pl [-r] [-d] [-f config-filename] [count addr]\n"; my %opts; # Note that this is used for both getops and the options file my @expected_opts = qw(active tunnelcfg bossname fsname type peer pubkeys privkeys); my $filename; my $remote; my $debug = 1; my $count; my $addr; my $active; my $type; my $tunnelcfg; my $ssh_port_fwds = ""; my $remote_script_dir; # location of the other sides fed-tun.pl my $event_repeater; # The pathname of the event repeater my $remote_config_file; # Config file for the other side if ($#ARGV != 0 && !getopts('df:r', \%opts)) { die "$usage"; } if (defined($opts{'d'})) { $debug = 1; } if (defined($opts{'f'})) { $filename = $opts{'f'}; } if (defined($opts{'r'})) { $remote = 1; die "$usage" if ($#ARGV != 1); $count = pop @ARGV; $addr = pop @ARGV; } die "$usage" if (!defined($remote) && !defined($filename)); if (defined($filename)) { &parse_config("$filename", \%opts) || die "Cannot read config file $filename: $!\n"; foreach my $opt (@expected_opts) { warn "Missing $opt option\n" if (!defined($opts{$opt})); } $active = 1 if ($opts{'active'} =~ /true/i); $tunnelcfg = 1 if ($opts{'tunnelcfg'} =~ /true/i); $type = $opts{'type'}; $type =~ tr/A-Z/a-z/; $remote_script_dir = $opts{'remotescriptdir'} || "."; $event_repeater = $opts{'eventrepeater'}; $remote_config_file = $opts{'remoteconfigfile'}; $remote_config_file = "-f $remote_config_file" if $remote_config_file; if (defined($opts{'fsname'})) { $ssh_port_fwds = "-R :$SMBFS_PORT:$opts{'fsname'}:$SMBFS_PORT "; } if (defined($opts{'bossname'})) { $ssh_port_fwds .= "-R :$TMCD_PORT:$opts{'bossname'}:$TMCD_PORT "; } if (defined($opts{'eventservername'})) { $ssh_port_fwds .= "-R :$remote_pubsub_port:$opts{'eventservername'}:" . "$PUBSUB_PORT "; } if (defined($opts{'remoteeventservername'})) { $ssh_port_fwds .= "-L :$remote_pubsub_port:" . "$opts{'remoteeventservername'}:$PUBSUB_PORT "; } $ssh_port_fwds = "" if ($opts{'type'} eq 'experiment'); print "ssh_port_fwds = $ssh_port_fwds\n" if ($debug); } # Both sides need to have GatewayPorts and PermitTunnel set. Copy the existing # sshd_config, making sure GatewayPorts and PermitTunnel are set to yes, # replace the original, and restart sshd. my $ports_on = 0; my $tunnel_on = 0; my $conf = new IO::File($sshd_config) || die "Can't open $sshd_config: $!\n"; my $new_conf = new IO::File(">/tmp/sshd_config") || die "Can't open new ssh_config: $!\n"; while(<$conf>) { s/^\s*GatewayPorts.*/GatewayPorts yes/ && do { print $new_conf $_ unless $ports_on++; next; }; s/^\s*PermitTunnel.*/PermitTunnel yes/ && do { print $new_conf $_ unless $tunnel_on++; next; }; print $new_conf $_; } print $new_conf "GatewayPorts yes\n" unless $ports_on; print $new_conf "PermitTunnel yes\n" unless $tunnel_on; $conf->close(); $new_conf->close(); copy("/tmp/sshd_config", $sshd_config) || die "Cannot replace $sshd_config: $!\n"; system("/etc/rc.d/sshd restart"); # Need these to make the Ethernet tap and bridge to work... system("kldload /boot/kernel/bridgestp.ko") if -r "/boot/kernel/bridgestp.ko"; system("kldload /boot/kernel/if_bridge.ko"); system("kldload /boot/kernel/if_tap.ko"); if (!$remote) { # At the Berkeley federant, the outgoing addresses are fixed and based on # the control net. We do set up a host route to the peer. This is a # simplified version of the DETER setup. &setup_tunnel_cfg(%opts); } if (!$remote) { system("umask 077 && cp $opts{'privkeys'} /root/.ssh/id_rsa"); system("umask 077 && cp $opts{'pubkeys'} /root/.ssh/id_rsa.pub"); system("umask 077 && cat $opts{'pubkeys'} >> /root/.ssh/authorized_keys"); } if ($active) { # If we're the initiator, open up a separate tunnel to the remote # host for each of the different experiment net interfaces on this # machine. Execute this startup script on the far end, but with # the -r option to indicate that it's getting invoked remotely and # we should just handle the remote-end tasks. # Set up synchronization, so that the various user machines won't try to # contact boss before the tunnels are set up. (Thanks jjh!) do { system("$NC -z $opts{'peer'} $SSH_PORT"); } until (!$?); # XXX: Do we need to clear out previously created bridge interfaces? my $count = 0; my @SSHCMD; # If we are just setting up a control net connection, just fire up # ssh with the null command, and hang loose. if ($type eq "control") { system("$SSH $ssh_port_fwds -Nno \"StrictHostKeyChecking no\" $opts{'peer'} &"); #or die "Failed to run ssh"; exit; } open(IFFILE, "/var/emulab/boot/ifmap") || die "couldn't open ifmap\n"; while () { my @a = split(' '); my $iface = $a[0]; my $addr = $a[1]; my $bridge = "bridge" . $count; my $tun = "tap" . $count; print "Found $iface, $addr, to bridge on $bridge\n" if ($debug); # Note that we're going to fire off an ssh which will never return; # we need the connection to stay up to keep the tunnel up. # In order to check for problems, we open it this way and read # the expected single line of output when the tunnel is connected. # The Tunnel option specifies the level to tunnel at. Ethernet creates # a tap device rather than a tun device. Strict host key checking # avoids asking the user to OK a strange host key. my $cmd = "$SSH -w $count:$count -o \"Tunnel ethernet\" " . "-o \"StrictHostKeyChecking no\" " . "$opts{'peer'} \"$remote_script_dir/fed-tun.pl " . "$remote_config_file -r $addr $count\" & |"; print "$cmd\n" if $debug; open($SSHCMD[$count], $cmd) or die "Failed to run ssh"; my $check = <$SSHCMD[$count]>; # Make sure something ran... print "Got line [$check] from remote side\n" if ($debug); &setup_bridging($tun, $bridge, $iface, $addr); $count++; $ssh_port_fwds = ""; # only do this on the first connection } close(IFFILE); # Start a local event repeater (unless we're missing parameters die "Missing event repeater params (No config file ?)\n" unless $event_repeater && $opts{'remoteexperiment'} && $opts{'localexperiment'}; print "Starting event repeater\n" if $debug; print("$event_repeater -M -P $remote_pubsub_port -S localhost " . "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}\n") if $debug; # Connect to the forwarded pubsub port on this host to listen to the # other experiment's events, and to the local experiment to forward # events. system("$event_repeater -M -P $remote_pubsub_port -S localhost " . "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}"); warn "Event repeater returned $?\n" if $?; } elsif ($remote) { # We're on the remote system; figure out which interface to # tweak, based on the IP address passed in. my $iter = 0; my $iface; open(RTFILE, "$ROUTE_GET $addr |") || die "couldn't do $ROUTE_GET\n"; while () { if (/interface: (\w+)/) { $iface = $1; } } close(RTFILE); die "Couldn't find interface to use." if (!defined($iface)); my $bridge = "bridge" . $count; my $tun = "tap" . $count; &setup_bridging($tun, $bridge, $iface, $addr); print "Remote connection all set up!\n"; # Trigger other end with output # If this is the first remote invocation on a control gateway, start the # event repeater. my $file = new IO::File(">/tmp/remote"); print($file "hello!!!\n") if $file; if ( $count == 0 && $type ne "experiment" ) { my $remote_pubsub_port = $PUBSUB_PORT - 1; # There will be a local # pubsubd running, so we # dodge the port on the # remote tunnel node. print($file "In here!\n"); # Make sure we have the relevant parameters die "Missing event repeater params (No config file ?)\n" unless $event_repeater && $opts{'remoteexperiment'} && $opts{'localexperiment'}; print "Starting event repeater\n" if $debug; print($file "$event_repeater -P $remote_pubsub_port -S localhost " . "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}\n") if $file; # Connect to the forwarded pubsub port on this host to listen to the # other experiment's events, and to the local experiment to forward # events. system("$event_repeater -P $remote_pubsub_port -S localhost " . "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}"); warn "Event repeater returned $?\n" if $?; } $file->close() if $file; } else { print "inactive end of a connection, finishing" if ($debug); } print "all done!\n" if ($debug); exit; # Set up the bridging for the new stuff... sub setup_bridging($; $; $; $) { my ($tun, $bridge, $iface, $addr) = @_; print "Waiting to see if new iface $tun is up\n" if ($debug); do { sleep 1; system("$IFCONFIG $tun"); } until (!$?); print "setting up $bridge with $iface and nuking $addr\n" if ($debug); system("ifconfig $bridge create"); system("ifconfig $iface delete $addr"); system("ifconfig $bridge addm $iface up"); system("ifconfig $bridge addm $tun"); } # Set up tunnel info for UCB testbed sub setup_tunnel_cfg { my (%opts) = @_; my $tunnel_iface = "em5"; my $control_iface = "em4"; my $tunnel_prefix = "192.154.6."; my $tunnel_router = "192.154.6.1"; my $netmask = "255.255.255.0"; my $ipipe = new IO::Pipe(); my $pcnum; $ipipe->reader("$IFCONFIG $control_iface"); while (<$ipipe>) { /inet\s+\d+\.\d+\.\d+\.(\d+)/ && do { $pcnum = $1; last; }; } $ipipe->close(); $pcnum += 50; $pcnum &=0xff if $pcnum > 255; system("ifconfig $tunnel_iface $tunnel_prefix$pcnum netmask $netmask"); warn "configuration of tunnel interface failed" if ($?); # Sometimes the insertion of DNS names lags a bit. Retry this # configuration a few times to let DNS catch up. Might want to really # check the DNS name before we try this... my $config_succeeded = 0; my $tries = 0; my $max_retries = 30; do { system("route add $opts{'peer'} $tunnel_router"); if ( $? ) { warn "configuration routes via tunnel interface failed"; $tries++; sleep(10); } else { $config_succeeded = 1; } } until ( $config_succeeded || $tries > $max_retries ); print "setup_tunnel_cfg done\n" if ($debug); } # Trick config-file parsing code from Ted Faber: # Parse the config file. The format is a colon-separated parameter name # followed by the value of that parameter to the end of the line. This parses # that format and puts the parameters into the referenced hash. Parameter # names are mapped to lower case, parameter values are unchanged. Returns 0 on # failure (e.g. file open) and 1 on success. sub parse_config { my($file, $href) = @_; my($fh) = new IO::File($file); unless ($fh) { warn "Can't open $file: $!\n"; return 0; } while (<$fh>) { next if /^\s*#/ || /^\s*$/; # Skip comments & blanks chomp; /^([^:]+):\s*(.*)/ && do { my($key) = $1; $key =~ tr/A-Z/a-z/; $href->{$key} = $2; next; }; warn "Unparasble line in $file: $_\n"; } $fh->close(); # It will close when it goes out of scope, but... return 1; }