[c9f5490] | 1 | #!/usr/bin/perl -w |
---|
| 2 | |
---|
| 3 | # Kevin Lahey, lahey@isi.edu |
---|
| 4 | # July 11, 2007 |
---|
| 5 | |
---|
| 6 | # Set up ssh tunnel infrastructure for federation: |
---|
| 7 | # |
---|
[a098fab] | 8 | # * Parse the configuration file provided. |
---|
| 9 | # |
---|
| 10 | # * Figure out whether we're the initiator or the reciever; if we're |
---|
| 11 | # the receiver, we just need to set up ssh keys and exit. |
---|
| 12 | # |
---|
| 13 | # * Pick out the experimental interface, remove the IP address |
---|
[c9f5490] | 14 | # |
---|
[a098fab] | 15 | # * Create a layer 2 ssh tunnel, set up bridging, and hang loose. |
---|
[fd7a59b] | 16 | # |
---|
[c9f5490] | 17 | |
---|
| 18 | use strict; |
---|
| 19 | use Getopt::Std; |
---|
| 20 | use POSIX qw(strftime); |
---|
| 21 | use Sys::Hostname; |
---|
[a098fab] | 22 | use IO::File; |
---|
[c9f5490] | 23 | |
---|
[d501c38] | 24 | my $IFCONFIG = "/sbin/ifconfig"; |
---|
[c9f5490] | 25 | my $TMCC = "/usr/local/etc/emulab/tmcc"; |
---|
| 26 | my $SSH = "/usr/local/bin/ssh"; |
---|
[a098fab] | 27 | my $NC = "/usr/bin/nc"; |
---|
| 28 | my $SSH_PORT = 22; |
---|
[fd7a59b] | 29 | my $ROUTE_GET = "/sbin/route get"; # XXX: works on FreeBSD, but should |
---|
| 30 | # should be 'ip route get' for Linux |
---|
[c9f5490] | 31 | |
---|
[f64fa81] | 32 | # Ports that are forwarded between testbeds |
---|
| 33 | my $TMCD_PORT = 7777; |
---|
| 34 | my $SMBFS_PORT = 139; |
---|
| 35 | my $PUBSUB_PORT = 16505; |
---|
| 36 | |
---|
[a098fab] | 37 | sub setup_bridging; |
---|
| 38 | sub setup_tunnel_cfg; |
---|
| 39 | sub parse_config; |
---|
[c9f5490] | 40 | |
---|
[a098fab] | 41 | # Option use is as follows: |
---|
| 42 | # -f filename containing config file |
---|
| 43 | # -d turn on debugging output |
---|
| 44 | # -r remotely invoked |
---|
| 45 | # (if remotely invoked the last two args are the tun interface |
---|
| 46 | # number and the remote address of the tunnel) |
---|
[c9f5490] | 47 | |
---|
[a098fab] | 48 | my $usage = "Usage: fed-tun.pl [-r] [-d] [-f config-filename] [count addr]\n"; |
---|
[c9f5490] | 49 | |
---|
[fd7a59b] | 50 | my %opts; # Note that this is used for both getops and the options file |
---|
[cd3eceb] | 51 | my @expected_opts = qw(active tunnelcfg bossname fsname type |
---|
| 52 | peer pubkeys privkeys); |
---|
[a098fab] | 53 | my $filename; |
---|
| 54 | my $remote; |
---|
| 55 | my $debug = 1; |
---|
[c9f5490] | 56 | my $count; |
---|
| 57 | my $addr; |
---|
[cd3eceb] | 58 | my $active; |
---|
| 59 | my $type; |
---|
| 60 | my $tunnelcfg; |
---|
| 61 | my $ssh_port_fwds = ""; |
---|
[c8c45ee] | 62 | my $remote_script_dir; # location of the other sides fed-tun.pl |
---|
[8034579] | 63 | my $event_repeater; # The pathname of the event repeater |
---|
| 64 | my $remote_config_file; # Config file for the other side |
---|
[c9f5490] | 65 | |
---|
[a098fab] | 66 | if ($#ARGV != 0 && !getopts('df:r', \%opts)) { |
---|
[c9f5490] | 67 | die "$usage"; |
---|
| 68 | } |
---|
| 69 | |
---|
[a098fab] | 70 | if (defined($opts{'d'})) { |
---|
| 71 | $debug = 1; |
---|
| 72 | } |
---|
[c9f5490] | 73 | |
---|
[a098fab] | 74 | if (defined($opts{'f'})) { |
---|
| 75 | $filename = $opts{'f'}; |
---|
[c9f5490] | 76 | } |
---|
| 77 | |
---|
| 78 | if (defined($opts{'r'})) { |
---|
| 79 | $remote = 1; |
---|
| 80 | die "$usage" if ($#ARGV != 1); |
---|
| 81 | |
---|
| 82 | $count = pop @ARGV; |
---|
| 83 | $addr = pop @ARGV; |
---|
| 84 | } |
---|
| 85 | |
---|
[a098fab] | 86 | die "$usage" if (!defined($remote) && !defined($filename)); |
---|
[c9f5490] | 87 | |
---|
[a098fab] | 88 | if (defined($filename)) { |
---|
| 89 | &parse_config("$filename", \%opts) || |
---|
| 90 | die "Cannot read config file $filename: $!\n"; |
---|
[c9f5490] | 91 | |
---|
[cd3eceb] | 92 | foreach my $opt (@expected_opts) { |
---|
| 93 | warn "Missing $opt option\n" if (!defined($opts{$opt})); |
---|
| 94 | } |
---|
| 95 | |
---|
| 96 | $active = 1 if ($opts{'active'} =~ /true/i); |
---|
| 97 | $tunnelcfg = 1 if ($opts{'tunnelcfg'} =~ /true/i); |
---|
| 98 | $type = $opts{'type'}; |
---|
| 99 | $type =~ tr/A-Z/a-z/; |
---|
[c8c45ee] | 100 | $remote_script_dir = $opts{'remotescriptdir'} || "."; |
---|
[8034579] | 101 | $event_repeater = $opts{'eventrepeater'}; |
---|
| 102 | $remote_config_file = $opts{'remoteconfigfile'}; |
---|
| 103 | $remote_config_file = "-f $remote_config_file" if $remote_config_file; |
---|
[c9f5490] | 104 | |
---|
[cd3eceb] | 105 | if (defined($opts{'fsname'})) { |
---|
[f64fa81] | 106 | $ssh_port_fwds = "-R :$SMBFS_PORT:$opts{'fsname'}:$SMBFS_PORT "; |
---|
[cd3eceb] | 107 | } |
---|
[c9f5490] | 108 | |
---|
[cd3eceb] | 109 | if (defined($opts{'bossname'})) { |
---|
[f64fa81] | 110 | my $remote_pubsub_port = $PUBSUB_PORT - 1; # There will be a local |
---|
| 111 | # pubsubd running, so we |
---|
| 112 | # dodge the port on the |
---|
| 113 | # remote tunnel node. |
---|
| 114 | $ssh_port_fwds .= "-R :$TMCD_PORT:$opts{'bossname'}:$TMCD_PORT "; |
---|
| 115 | $ssh_port_fwds .= "-R :$remote_pubsub_port:$opts{'eventservername'}:" . |
---|
| 116 | "$PUBSUB_PORT "; |
---|
[cd3eceb] | 117 | } |
---|
[c9f5490] | 118 | |
---|
[cd3eceb] | 119 | $ssh_port_fwds = "" if ($opts{'type'} eq 'experiment'); |
---|
[c9f5490] | 120 | |
---|
[cd3eceb] | 121 | print "ssh_port_fwds = $ssh_port_fwds\n" if ($debug); |
---|
| 122 | } |
---|
[c9f5490] | 123 | |
---|
[a098fab] | 124 | # Need these to make the Ethernet tap and bridge to work... |
---|
[c9f5490] | 125 | |
---|
[a098fab] | 126 | system("kldload /boot/kernel/if_bridge.ko"); |
---|
| 127 | system("kldload /boot/kernel/if_tap.ko"); |
---|
[c9f5490] | 128 | |
---|
[cd3eceb] | 129 | if ($tunnelcfg && !$remote) { |
---|
[a098fab] | 130 | # Most Emulab-like testbeds use globally-routable addresses on the |
---|
| 131 | # control net; at DETER, we isolate the control net from the Internet. |
---|
| 132 | # On DETER-like testbeds, we need to create special tunneling nodes |
---|
| 133 | # with external access. Set up the external addresses as necessary. |
---|
| 134 | &setup_tunnel_cfg(%opts); |
---|
[c9f5490] | 135 | } |
---|
| 136 | |
---|
| 137 | if (!$remote) { |
---|
[a098fab] | 138 | system("umask 077 && cp $opts{'privkeys'} /root/.ssh/id_rsa"); |
---|
| 139 | system("umask 077 && cp $opts{'pubkeys'} /root/.ssh/id_rsa.pub"); |
---|
| 140 | system("umask 077 && cat $opts{'pubkeys'} >> /root/.ssh/authorized_keys"); |
---|
| 141 | } |
---|
[c9f5490] | 142 | |
---|
[cd3eceb] | 143 | if ($active) { |
---|
[a098fab] | 144 | # If we're the initiator, open up a separate tunnel to the remote |
---|
| 145 | # host for each of the different experiment net interfaces on this |
---|
| 146 | # machine. Execute this startup script on the far end, but with |
---|
| 147 | # the -r option to indicate that it's getting invoked remotely and |
---|
| 148 | # we should just handle the remote-end tasks. |
---|
[c9f5490] | 149 | |
---|
[a098fab] | 150 | # Set up synchronization, so that the various user machines won't try to |
---|
| 151 | # contact boss before the tunnels are set up. (Thanks jjh!) |
---|
[c9f5490] | 152 | |
---|
[a098fab] | 153 | do { |
---|
| 154 | system("$NC -z $opts{'peer'} $SSH_PORT"); |
---|
| 155 | } until (!$?); |
---|
[c9f5490] | 156 | |
---|
| 157 | # XXX: Do we need to clear out previously created bridge interfaces? |
---|
| 158 | |
---|
| 159 | my $count = 0; |
---|
[a098fab] | 160 | my @SSHCMD; |
---|
[c9f5490] | 161 | |
---|
[d501c38] | 162 | # If we are just setting up a control net connection, just fire up |
---|
| 163 | # ssh with the null command, and hang loose. |
---|
| 164 | |
---|
| 165 | if ($type eq "control") { |
---|
| 166 | system("$SSH $ssh_port_fwds -Nno \"StrictHostKeyChecking no\" $opts{'peer'} &"); #or die "Failed to run ssh"; |
---|
| 167 | |
---|
| 168 | exit; |
---|
| 169 | } |
---|
| 170 | |
---|
[c9f5490] | 171 | open(IFFILE, "/var/emulab/boot/ifmap") || die "couldn't open ifmap\n"; |
---|
| 172 | while (<IFFILE>) { |
---|
| 173 | my @a = split(' '); |
---|
| 174 | my $iface = $a[0]; |
---|
| 175 | my $addr = $a[1]; |
---|
| 176 | my $bridge = "bridge" . $count; |
---|
| 177 | my $tun = "tap" . $count; |
---|
| 178 | |
---|
| 179 | print "Found $iface, $addr, to bridge on $bridge\n" if ($debug); |
---|
| 180 | |
---|
[a098fab] | 181 | # Note that we're going to fire off an ssh which will never return; |
---|
| 182 | # we need the connection to stay up to keep the tunnel up. |
---|
| 183 | # In order to check for problems, we open it this way and read |
---|
| 184 | # the expected single line of output when the tunnel is connected. |
---|
[c9f5490] | 185 | |
---|
[8034579] | 186 | print "$SSH -w $count:$count $ssh_port_fwds -o \"StrictHostKeyChecking no\" $opts{'peer'} \"$remote_script_dir/fed-tun.pl $remote_config_file -r $addr $count\"\n" if $debug; |
---|
| 187 | open($SSHCMD[$count], "$SSH -w $count:$count $ssh_port_fwds -o \"StrictHostKeyChecking no\" $opts{'peer'} \"$remote_script_dir/fed-tun.pl $remote_config_file -r $addr $count\" |") or die "Failed to run ssh"; |
---|
[c9f5490] | 188 | |
---|
[a098fab] | 189 | my $check = <$SSHCMD[$count]>; # Make sure something ran... |
---|
[d501c38] | 190 | print "Got line [$check] from remote side\n" if ($debug); |
---|
[c9f5490] | 191 | |
---|
[a098fab] | 192 | &setup_bridging($tun, $bridge, $iface, $addr); |
---|
[c9f5490] | 193 | $count++; |
---|
[a098fab] | 194 | $ssh_port_fwds = ""; # only do this on the first connection |
---|
[c9f5490] | 195 | } |
---|
| 196 | close(IFFILE); |
---|
[a098fab] | 197 | } elsif ($remote) { |
---|
[fd7a59b] | 198 | # We're on the remote system; figure out which interface to |
---|
| 199 | # tweak, based on the IP address passed in. |
---|
[c9f5490] | 200 | |
---|
| 201 | my $iter = 0; |
---|
[fd7a59b] | 202 | my $iface; |
---|
[c9f5490] | 203 | |
---|
[fd7a59b] | 204 | open(RTFILE, "$ROUTE_GET $addr |") || die "couldn't do $ROUTE_GET\n"; |
---|
| 205 | while (<RTFILE>) { |
---|
| 206 | if (/interface: (\w+)/) { |
---|
| 207 | $iface = $1; |
---|
| 208 | } |
---|
[c9f5490] | 209 | } |
---|
[fd7a59b] | 210 | close(RTFILE); |
---|
| 211 | |
---|
| 212 | die "Couldn't find interface to use." if (!defined($iface)); |
---|
| 213 | my $bridge = "bridge" . $count; |
---|
| 214 | my $tun = "tap" . $count; |
---|
[a098fab] | 215 | |
---|
[fd7a59b] | 216 | &setup_bridging($tun, $bridge, $iface, $addr); |
---|
[a098fab] | 217 | print "Remote connection all set up!\n"; # Trigger other end with output |
---|
[8034579] | 218 | |
---|
| 219 | # If this is the first remote invocation on a control gateway, start the |
---|
| 220 | # event repeater. |
---|
| 221 | |
---|
| 222 | my $file = new IO::File(">/tmp/remote"); |
---|
| 223 | print($file "hello!!!\n") if $file; |
---|
| 224 | if ( $count == 0 && $type ne "experiment" ) { |
---|
| 225 | my $remote_pubsub_port = $PUBSUB_PORT - 1; # There will be a local |
---|
| 226 | # pubsubd running, so we |
---|
| 227 | # dodge the port on the |
---|
| 228 | # remote tunnel node. |
---|
| 229 | print($file "In here!\n"); |
---|
| 230 | # Make sure we have the relevant parameters |
---|
| 231 | die "Missing event repeater params (No config file ?)\n" |
---|
| 232 | unless $event_repeater && $opts{'remoteexperiment'} && |
---|
| 233 | $opts{'localexperiment'}; |
---|
| 234 | |
---|
| 235 | print "Starting event repeater\n" if $debug; |
---|
| 236 | |
---|
| 237 | print($file "$event_repeater -P $remote_pubsub_port -S localhost " . |
---|
| 238 | "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}\n") |
---|
| 239 | if $file; |
---|
| 240 | # Connect to the forwarded pubsub port on this host to listen to the |
---|
| 241 | # other experiment's events, and to the local experiment to forward |
---|
| 242 | # events. |
---|
| 243 | system("$event_repeater -P $remote_pubsub_port -S localhost " . |
---|
| 244 | "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}"); |
---|
| 245 | warn "Event repeater returned $?\n" if $?; |
---|
| 246 | } |
---|
| 247 | $file->close() if $file; |
---|
[a098fab] | 248 | } else { |
---|
| 249 | print "inactive end of a connection, finishing" if ($debug); |
---|
[c9f5490] | 250 | } |
---|
| 251 | |
---|
| 252 | print "all done!\n" if ($debug); |
---|
| 253 | exit; |
---|
| 254 | |
---|
| 255 | |
---|
| 256 | # Set up the bridging for the new stuff... |
---|
| 257 | |
---|
[a098fab] | 258 | sub setup_bridging($; $; $; $) { |
---|
[c9f5490] | 259 | my ($tun, $bridge, $iface, $addr) = @_; |
---|
| 260 | |
---|
[d501c38] | 261 | print "Waiting to see if new iface $tun is up\n" if ($debug); |
---|
| 262 | |
---|
| 263 | do { |
---|
| 264 | sleep 1; |
---|
| 265 | system("$IFCONFIG $tun"); |
---|
| 266 | } until (!$?); |
---|
| 267 | |
---|
[c9f5490] | 268 | print "setting up $bridge with $iface and nuking $addr\n" if ($debug); |
---|
| 269 | |
---|
| 270 | system("ifconfig $bridge create"); |
---|
| 271 | system("ifconfig $iface delete $addr"); |
---|
| 272 | system("ifconfig $bridge addm $iface up"); |
---|
| 273 | system("ifconfig $bridge addm $tun"); |
---|
| 274 | } |
---|
[a098fab] | 275 | |
---|
| 276 | # Set up tunnel info for DETER-like testbeds. |
---|
| 277 | |
---|
| 278 | sub setup_tunnel_cfg { |
---|
| 279 | my (%opts) = @_; |
---|
| 280 | my $tunnel_iface = "em0"; # XXX |
---|
| 281 | my $tunnel_ip; |
---|
| 282 | my $tunnel_mask; |
---|
| 283 | my $tunnel_mac; |
---|
| 284 | my $tunnel_router; |
---|
| 285 | |
---|
| 286 | print "Opening $TMCC tunnelip\n" if ($debug); |
---|
| 287 | |
---|
| 288 | open(TMCD, "$TMCC tunnelip |") || die "tmcc failed\n"; |
---|
| 289 | print "Opened $TMCC tunnelip\n" if ($debug); |
---|
| 290 | while (<TMCD>) { |
---|
| 291 | print "got one line from tmcc\n" if ($debug); |
---|
| 292 | print if ($debug); |
---|
| 293 | if (/^TUNNELIP=([0-9.]*) TUNNELMASK=([0-9.]*) TUNNELMAC=(\w*) TUNNELROUTER=([0-9.]*)$/) { |
---|
| 294 | $tunnel_ip = $1; |
---|
| 295 | $tunnel_mask = $2; |
---|
| 296 | $tunnel_mac = $3; |
---|
| 297 | $tunnel_router = $4; |
---|
| 298 | } |
---|
| 299 | } |
---|
| 300 | close(TMCD); |
---|
| 301 | |
---|
| 302 | die "Unable to determine tunnel node configuration information" |
---|
| 303 | if (!defined($tunnel_router)); |
---|
| 304 | |
---|
| 305 | print "tunnel options: ip=$tunnel_ip mask=$tunnel_mask mac=$tunnel_mac router=$tunnel_router\n" if ($debug); |
---|
| 306 | |
---|
| 307 | # Sadly, we ignore the tunnel mac for now -- we should eventually |
---|
| 308 | # use it to determine which interface to use, just like the |
---|
| 309 | # Emulab startup scripts. |
---|
| 310 | |
---|
| 311 | system("ifconfig $tunnel_iface $tunnel_ip" . |
---|
| 312 | ($tunnel_mask ? " netmask $tunnel_mask" : "")); |
---|
| 313 | warn "configuration of tunnel interface failed" if ($?); |
---|
[4abace9] | 314 | |
---|
| 315 | # Sometimes the insertion of DNS names lags a bit. Retry this |
---|
| 316 | # configuration a few times to let DNS catch up. Might want to really |
---|
| 317 | # check the DNS name before we try this... |
---|
| 318 | my $config_succeeded = 0; |
---|
| 319 | my $tries = 0; |
---|
| 320 | my $max_retries = 6; |
---|
| 321 | |
---|
| 322 | do { |
---|
| 323 | system("route add $opts{'peer'} $tunnel_router"); |
---|
| 324 | if ( $? ) { |
---|
| 325 | warn "configuration routes via tunnel interface failed"; |
---|
| 326 | $tries++; |
---|
| 327 | sleep(10); |
---|
| 328 | } |
---|
| 329 | else { $config_succeeded = 1; } |
---|
| 330 | } until ( $config_succeeded || $tries > $max_retries ); |
---|
[a098fab] | 331 | |
---|
| 332 | print "setup_tunnel_cfg done\n" if ($debug); |
---|
| 333 | } |
---|
| 334 | |
---|
| 335 | # Trick config-file parsing code from Ted Faber: |
---|
| 336 | |
---|
| 337 | # Parse the config file. The format is a colon-separated parameter name |
---|
| 338 | # followed by the value of that parameter to the end of the line. This parses |
---|
| 339 | # that format and puts the parameters into the referenced hash. Parameter |
---|
| 340 | # names are mapped to lower case, parameter values are unchanged. Returns 0 on |
---|
| 341 | # failure (e.g. file open) and 1 on success. |
---|
| 342 | sub parse_config { |
---|
| 343 | my($file, $href) = @_; |
---|
| 344 | my($fh) = new IO::File($file); |
---|
| 345 | |
---|
| 346 | unless ($fh) { |
---|
| 347 | warn "Can't open $file: $!\n"; |
---|
| 348 | return 0; |
---|
| 349 | } |
---|
| 350 | |
---|
| 351 | while (<$fh>) { |
---|
| 352 | next if /^\s*#/ || /^\s*$/; # Skip comments & blanks |
---|
| 353 | chomp; |
---|
| 354 | /^([^:]+):\s*(.*)/ && do { |
---|
| 355 | my($key) = $1; |
---|
| 356 | |
---|
| 357 | $key =~ tr/A-Z/a-z/; |
---|
| 358 | $href->{$key} = $2; |
---|
| 359 | next; |
---|
| 360 | }; |
---|
| 361 | warn "Unparasble line in $file: $_\n"; |
---|
| 362 | } |
---|
| 363 | $fh->close(); # It will close when it goes out of scope, but... |
---|
| 364 | return 1; |
---|
| 365 | } |
---|