source: fedkit/fed-tun.ucb.pl @ d81971a

#!/usr/bin/perl -w

# Kevin Lahey, lahey@isi.edu
# July 11, 2007

# Set up ssh tunnel infrastructure for federation:
# 
# * Parse the configuration file provided.
#
# * Figure out whether we're the initiator or the reciever;  if we're 
#   the receiver, we just need to set up ssh keys and exit.
#
# * Pick out the experimental interface, remove the IP address
# 
# * Create a layer 2 ssh tunnel, set up bridging, and hang loose.
#

use strict;
use Getopt::Std;
use POSIX qw(strftime);
use Sys::Hostname;
use IO::File;
use IO::Pipe;
use File::Copy;

my $IFCONFIG = "/sbin/ifconfig";
my $TMCC = "/usr/local/etc/emulab/tmcc";
# If a special version of ssh is required, it will be installed in
# /usr/local/bin/ssh.  Otherwise use the usual one.
my $SSH = -x "/usr/local/bin/ssh" ? "/usr/local/bin/ssh" : "/usr/bin/ssh";
my $NC = "/usr/bin/nc";
my $SSH_PORT = 22;
my $ROUTE_GET = "/sbin/route get";  # XXX:  works on FreeBSD, but should
                                    #       should be 'ip route get' for Linux
my $sshd_config = "/etc/ssh/sshd_config";    # Probably should be a param

# Ports that are forwarded between testbeds
my $TMCD_PORT = 7777;
my $SMBFS_PORT = 139;
my $PUBSUB_PORT = 16505;


my $remote_pubsub_port = $PUBSUB_PORT - 1;  # There will be a local
                                            # pubsubd running, so we
                                            # dodge the port on the
                                            # remote tunnel node.
                                            
die "Cannot exec $SSH" unless -x $SSH;

sub setup_bridging;
sub setup_tunnel_cfg;
sub parse_config;

# Option use is as follows:
#    -f         filename containing config file
#    -d         turn on debugging output
#    -r         remotely invoked
#               (if remotely invoked the last two args are the tun interface
#               number and the remote address of the tunnel)

my $usage = "Usage: fed-tun.pl [-r] [-d] [-f config-filename] [count addr]\n";

my %opts;       # Note that this is used for both getops and the options file
my @expected_opts = qw(active tunnelcfg bossname fsname type 
                       peer pubkeys privkeys);
my $filename;
my $remote;
my $debug = 1;
my $count;
my $addr;
my $active;
my $type;
my $tunnelcfg;
my $ssh_port_fwds = "";
my $remote_script_dir;          # location of the other sides fed-tun.pl
my $event_repeater;             # The pathname of the event repeater
my $remote_config_file;         # Config file for the other side

if ($#ARGV != 0 && !getopts('df:r', \%opts)) {
    die "$usage";
}

if (defined($opts{'d'})) {
    $debug = 1;
}

if (defined($opts{'f'})) {
    $filename = $opts{'f'};
}

if (defined($opts{'r'})) {
    $remote = 1;
    die "$usage" if ($#ARGV != 1);
    
    $count = pop @ARGV;
    $addr = pop @ARGV;
}

die "$usage" if (!defined($remote) && !defined($filename));

if (defined($filename)) {
    &parse_config("$filename", \%opts) || 
        die "Cannot read config file $filename: $!\n";

    foreach my $opt (@expected_opts) {
        warn "Missing $opt option\n" if (!defined($opts{$opt}));
    }

    $active = 1 if ($opts{'active'} =~ /true/i);
    $tunnelcfg = 1 if ($opts{'tunnelcfg'} =~ /true/i);
    $type = $opts{'type'};
    $type =~ tr/A-Z/a-z/;
    $remote_script_dir = $opts{'remotescriptdir'} || ".";
    $event_repeater = $opts{'eventrepeater'};
    $remote_config_file = $opts{'remoteconfigfile'};
    $remote_config_file = "-f $remote_config_file" if $remote_config_file;

    if (defined($opts{'fsname'})) {
        $ssh_port_fwds = "-R :$SMBFS_PORT:$opts{'fsname'}:$SMBFS_PORT ";
    }

    if (defined($opts{'bossname'})) {
        $ssh_port_fwds .= "-R :$TMCD_PORT:$opts{'bossname'}:$TMCD_PORT ";
    }

    if (defined($opts{'eventservername'})) {
        $ssh_port_fwds .= "-R :$remote_pubsub_port:$opts{'eventservername'}:" . 
            "$PUBSUB_PORT ";
    }
    if (defined($opts{'remoteeventservername'})) {
        $ssh_port_fwds .= "-L :$remote_pubsub_port:" . 
            "$opts{'remoteeventservername'}:$PUBSUB_PORT ";
    }

    $ssh_port_fwds = "" if ($opts{'type'} eq 'experiment');

    print "ssh_port_fwds = $ssh_port_fwds\n" if ($debug);
}


# Both sides need to have GatewayPorts and PermitTunnel set.  Copy the existing
# sshd_config, making sure GatewayPorts and PermitTunnel are set to yes,
# replace the original, and restart sshd.
my $ports_on = 0;
my $tunnel_on = 0;

my $conf = new IO::File($sshd_config) || die "Can't open $sshd_config: $!\n";
my $new_conf = new IO::File(">/tmp/sshd_config") || 
    die "Can't open new ssh_config: $!\n";

while(<$conf>) {
    s/^\s*GatewayPorts.*/GatewayPorts yes/ && do {
        print $new_conf $_ unless $ports_on++;
        next;
    };
    s/^\s*PermitTunnel.*/PermitTunnel yes/ && do {
        print $new_conf $_ unless $tunnel_on++;
        next;
    };
    print $new_conf $_;
}
print $new_conf "GatewayPorts yes\n" unless $ports_on;
print $new_conf "PermitTunnel yes\n" unless $tunnel_on;
$conf->close();
$new_conf->close();

copy("/tmp/sshd_config", $sshd_config) || 
    die "Cannot replace $sshd_config: $!\n";

system("/etc/rc.d/sshd restart");


# Need these to make the Ethernet tap and bridge to work...
system("kldload /boot/kernel/bridgestp.ko") 
    if -r "/boot/kernel/bridgestp.ko";
system("kldload /boot/kernel/if_bridge.ko");
system("kldload /boot/kernel/if_tap.ko");

if (!$remote) {
    # At the Berkeley federant, the outgoing addresses are fixed and based on
    # the control net.  We do set up a host route to the peer.  This is a
    # simplified version of the DETER setup.

    &setup_tunnel_cfg(%opts);
}

if (!$remote) {
    system("umask 077 && cp $opts{'privkeys'} /root/.ssh/id_rsa");
    system("umask 077 && cp $opts{'pubkeys'} /root/.ssh/id_rsa.pub");
    system("umask 077 && cat $opts{'pubkeys'} >> /root/.ssh/authorized_keys");
}

if ($active) {
    # If we're the initiator, open up a separate tunnel to the remote 
    # host for each of the different experiment net interfaces on this 
    # machine.  Execute this startup script on the far end, but with 
    # the -r option to indicate that it's getting invoked remotely and 
    # we should just handle the remote-end tasks.

    # Set up synchronization, so that the various user machines won't try to
    # contact boss before the tunnels are set up.  (Thanks jjh!)

    do {
        system("$NC -z $opts{'peer'} $SSH_PORT");
    } until (!$?);

    # XXX:  Do we need to clear out previously created bridge interfaces?

    my $count = 0;
    my @SSHCMD;

    # If we are just setting up a control net connection, just fire up
    # ssh with the null command, and hang loose.

    if ($type eq "control") {
        system("$SSH $ssh_port_fwds -Nno \"StrictHostKeyChecking no\" $opts{'peer'} &"); #or die "Failed to run ssh";

        exit;
    }

    open(IFFILE, "/var/emulab/boot/ifmap") || die "couldn't open ifmap\n";
    while (<IFFILE>) {
        my @a = split(' ');
        my $iface = $a[0];
        my $addr = $a[1];
        my $bridge = "bridge" . $count;
        my $tun = "tap" . $count;

        print "Found $iface, $addr, to bridge on $bridge\n" if ($debug);

        # Note that we're going to fire off an ssh which will never return;
        # we need the connection to stay up to keep the tunnel up.
        # In order to check for problems, we open it this way and read
        # the expected single line of output when the tunnel is connected.

        # The Tunnel option specifies the level to tunnel at.  Ethernet creates
        # a tap device rather than a tun device.  Strict host key checking
        # avoids asking the user to OK a strange host key.
        my $cmd =  "$SSH -w $count:$count -o \"Tunnel ethernet\" " . 
            "-o \"StrictHostKeyChecking no\" " . 
            "$opts{'peer'}  \"$remote_script_dir/fed-tun.pl " . 
            "$remote_config_file -r $addr $count\" & |";

        print "$cmd\n" if $debug;
        open($SSHCMD[$count], $cmd) or die "Failed to run ssh";

        my $check = <$SSHCMD[$count]>;  # Make sure something ran...
        print "Got line [$check] from remote side\n" if ($debug);

        &setup_bridging($tun, $bridge, $iface, $addr);
        $count++;
        $ssh_port_fwds = "";  # only do this on the first connection
    }
    close(IFFILE);

    # Start a local event repeater (unless we're missing parameters 
    die "Missing event repeater params (No config file ?)\n"
        unless $event_repeater && $opts{'remoteexperiment'} && 
        $opts{'localexperiment'};

    print "Starting event repeater\n" if $debug;
    
    print("$event_repeater -M -P $remote_pubsub_port -S localhost " . 
        "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}\n") 
        if $debug;
    # Connect to the forwarded pubsub port on this host to listen to the
    # other experiment's events, and to the local experiment to forward
    # events.
    system("$event_repeater -M -P $remote_pubsub_port -S localhost " . 
        "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}");
    warn "Event repeater returned $?\n" if $?;
} elsif ($remote) {
    # We're on the remote system;  figure out which interface to 
    # tweak, based on the IP address passed in.

    my $iter = 0;
    my $iface;

    open(RTFILE, "$ROUTE_GET $addr |") || die "couldn't do $ROUTE_GET\n";
    while (<RTFILE>) {
        if (/interface: (\w+)/) {
            $iface = $1;
        }
    }
    close(RTFILE);

    die "Couldn't find interface to use." if (!defined($iface));
    my $bridge = "bridge" . $count;
    my $tun = "tap" . $count;

    &setup_bridging($tun, $bridge, $iface, $addr);
    print "Remote connection all set up!\n";  # Trigger other end with output

    # If this is the first remote invocation on a control gateway, start the
    # event repeater.

    my $file = new IO::File(">/tmp/remote");
    print($file "hello!!!\n") if $file;
    if ( $count == 0 && $type ne "experiment" ) {
        my $remote_pubsub_port = $PUBSUB_PORT - 1;  # There will be a local
                                                    # pubsubd running, so we
                                                    # dodge the port on the
                                                    # remote tunnel node.
        print($file "In here!\n");
        # Make sure we have the relevant parameters
        die "Missing event repeater params (No config file ?)\n"
            unless $event_repeater && $opts{'remoteexperiment'} && 
            $opts{'localexperiment'};

        print "Starting event repeater\n" if $debug;
        
        print($file "$event_repeater -P $remote_pubsub_port -S localhost " . 
            "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}\n") 
            if $file;
        # Connect to the forwarded pubsub port on this host to listen to the
        # other experiment's events, and to the local experiment to forward
        # events.
        system("$event_repeater -P $remote_pubsub_port -S localhost " . 
            "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}");
        warn "Event repeater returned $?\n" if $?;
    }
    $file->close() if $file;
} else {
    print "inactive end of a connection, finishing" if ($debug);
}

print "all done!\n" if ($debug);
exit;


# Set up the bridging for the new stuff...

sub setup_bridging($; $; $; $) {
    my ($tun, $bridge, $iface, $addr) = @_;

    print "Waiting to see if new iface $tun is up\n" if ($debug);

    do {
        sleep 1;
        system("$IFCONFIG $tun");
    } until (!$?);

    print "setting up $bridge with $iface and nuking $addr\n" if ($debug);

    system("ifconfig $bridge create");
    system("ifconfig $iface delete $addr");
    system("ifconfig $bridge addm $iface up");
    system("ifconfig $bridge addm $tun");
}

# Set up tunnel info for UCB testbed

sub setup_tunnel_cfg {
    my (%opts) = @_;
    my $tunnel_iface = "em5";
    my $control_iface = "em4";
    my $tunnel_prefix = "192.154.6.";
    my $tunnel_router = "192.154.6.1";
    my $netmask = "255.255.255.0";
    my $ipipe = new IO::Pipe();
    my $pcnum;

    $ipipe->reader("$IFCONFIG $control_iface");

    while (<$ipipe>) {
        /inet\s+\d+\.\d+\.\d+\.(\d+)/ && do {
            $pcnum = $1;
            last;
        };
    }
    $ipipe->close();
    $pcnum += 50;
    $pcnum &=0xff if $pcnum > 255;

    system("ifconfig $tunnel_iface $tunnel_prefix$pcnum netmask $netmask");
    warn "configuration of tunnel interface failed" if ($?);

    # Sometimes the insertion of DNS names lags a bit.  Retry this
    # configuration a few times to let DNS catch up.  Might want to really
    # check the DNS name before we try this...
    my $config_succeeded = 0;
    my $tries = 0;
    my $max_retries = 30;

    do {
        system("route add $opts{'peer'} $tunnel_router");
        if ( $? ) {
            warn "configuration routes via tunnel interface failed";
            $tries++;
            sleep(10);
        }
        else { $config_succeeded = 1; }
    } until ( $config_succeeded || $tries > $max_retries );

    print "setup_tunnel_cfg done\n" if ($debug);
}

# Trick config-file parsing code from Ted Faber:

# Parse the config file.  The format is a colon-separated parameter name
# followed by the value of that parameter to the end of the line.  This parses
# that format and puts the parameters into the referenced hash.  Parameter
# names are mapped to lower case, parameter values are unchanged.  Returns 0 on
# failure (e.g. file open) and 1 on success.
sub parse_config {
    my($file, $href) = @_;
    my($fh) = new IO::File($file);
        
    unless ($fh) {
        warn "Can't open $file: $!\n";
        return 0;
    }

    while (<$fh>) {
        next if /^\s*#/ || /^\s*$/;     # Skip comments & blanks
        chomp;
        /^([^:]+):\s*(.*)/ && do {
            my($key) = $1; 

            $key =~ tr/A-Z/a-z/;
            $href->{$key} = $2;
            next;
        };
        warn "Unparasble line in $file: $_\n";
    }
    $fh->close();   # It will close when it goes out of scope, but...
    return 1;
}