1 | #!/usr/bin/perl -w |
---|
2 | |
---|
3 | # Kevin Lahey, lahey@isi.edu |
---|
4 | # July 11, 2007 |
---|
5 | |
---|
6 | # Set up ssh tunnel infrastructure for federation: |
---|
7 | # |
---|
8 | # * Parse the configuration file provided. |
---|
9 | # |
---|
10 | # * Figure out whether we're the initiator or the reciever; if we're |
---|
11 | # the receiver, we just need to set up ssh keys and exit. |
---|
12 | # |
---|
13 | # * Pick out the experimental interface, remove the IP address |
---|
14 | # |
---|
15 | # * Create a layer 2 ssh tunnel, set up bridging, and hang loose. |
---|
16 | # |
---|
17 | |
---|
18 | use strict; |
---|
19 | use Getopt::Std; |
---|
20 | use POSIX qw(strftime); |
---|
21 | use Sys::Hostname; |
---|
22 | use IO::File; |
---|
23 | use File::Copy; |
---|
24 | |
---|
25 | my $IFCONFIG = "/sbin/ifconfig"; |
---|
26 | my $TMCC = "/usr/local/etc/emulab/tmcc"; |
---|
27 | my $FINDIF = "/usr/local/etc/emulab/findif"; |
---|
28 | # If a special version of ssh is required, it will be installed in |
---|
29 | # /usr/local/bin/ssh. Otherwise use the usual one. |
---|
30 | my $SSH = -x "/usr/local/bin/ssh" ? "/usr/local/bin/ssh" : "/usr/bin/ssh"; |
---|
31 | my $NC = "/usr/bin/nc"; |
---|
32 | my $SSH_PORT = 22; |
---|
33 | my $ROUTE_GET = "/sbin/route get"; # XXX: works on FreeBSD, but should |
---|
34 | # should be 'ip route get' for Linux |
---|
35 | my $sshd_config = "/etc/ssh/sshd_config"; # Probably should be a param |
---|
36 | |
---|
37 | # Ports that are forwarded between testbeds |
---|
38 | my $TMCD_PORT = 7777; |
---|
39 | my $SMBFS_PORT = 139; |
---|
40 | my $PUBSUB_PORT = 16505; |
---|
41 | my $SEER_PORT = 16606; |
---|
42 | |
---|
43 | |
---|
44 | my $remote_pubsub_port = $PUBSUB_PORT - 1; # There will be a local |
---|
45 | # pubsubd running, so we |
---|
46 | # dodge the port on the |
---|
47 | # remote tunnel node. |
---|
48 | die "Cannot exec $SSH" unless -x $SSH; |
---|
49 | |
---|
50 | sub setup_bridging; |
---|
51 | sub setup_tunnel_cfg; |
---|
52 | sub parse_config; |
---|
53 | |
---|
54 | # Option use is as follows: |
---|
55 | # -f filename containing config file |
---|
56 | # -d turn on debugging output |
---|
57 | # -r remotely invoked |
---|
58 | # (if remotely invoked the last two args are the tun interface |
---|
59 | # number and the remote address of the tunnel) |
---|
60 | |
---|
61 | my $usage = "Usage: fed-tun.pl [-r] [-d] [-f config-filename] [count addr]\n"; |
---|
62 | |
---|
63 | my %opts; # Note that this is used for both getops and the options file |
---|
64 | my @expected_opts = qw(active tunnelcfg bossname fsname type |
---|
65 | peer pubkeys privkeys); |
---|
66 | my $filename; |
---|
67 | my $remote; |
---|
68 | my $debug = 1; |
---|
69 | my $count; |
---|
70 | my $addr; |
---|
71 | my $active; |
---|
72 | my $type; |
---|
73 | my $tunnelcfg; |
---|
74 | my @ssh_port_fwds; # Queue of ssh portforwarders to start. The |
---|
75 | # -L or -R is in here. |
---|
76 | my $remote_script_dir; # location of the other sides fed-tun.pl |
---|
77 | my $event_repeater; # The pathname of the event repeater |
---|
78 | |
---|
79 | if ($#ARGV != 0 && !getopts('df:rn', \%opts)) { |
---|
80 | die "$usage"; |
---|
81 | } |
---|
82 | |
---|
83 | if (defined($opts{'d'})) { |
---|
84 | $debug = 1; |
---|
85 | } |
---|
86 | |
---|
87 | if (defined($opts{'f'})) { |
---|
88 | $filename = $opts{'f'}; |
---|
89 | } |
---|
90 | else { |
---|
91 | # Find the configuration file in the usual place, which depends on what |
---|
92 | # experiment and project we're in. |
---|
93 | my $pid; |
---|
94 | my $eid; |
---|
95 | |
---|
96 | open(TMCD, "$TMCC status |") || die "tmcc failed\n"; |
---|
97 | print "Opened $TMCC status\n" if ($debug); |
---|
98 | while (<TMCD>) { |
---|
99 | chomp; |
---|
100 | /ALLOCATED=([^\/]+)\/(\S+)/ && do { |
---|
101 | $pid = $1; |
---|
102 | $eid = $2; |
---|
103 | }; |
---|
104 | } |
---|
105 | close(TMCD); |
---|
106 | my $hn = `hostname`; |
---|
107 | chomp $hn; |
---|
108 | $filename = "/proj/$pid/exp/$eid/tmp/$hn.gw.conf" |
---|
109 | if $pid and $eid; |
---|
110 | } |
---|
111 | |
---|
112 | if (defined($opts{'r'})) { |
---|
113 | $remote = 1; |
---|
114 | die "$usage" if ($#ARGV != 1); |
---|
115 | |
---|
116 | $count = pop @ARGV; |
---|
117 | $addr = pop @ARGV; |
---|
118 | } |
---|
119 | |
---|
120 | die "$usage" if (!defined($remote) && !defined($filename)); |
---|
121 | |
---|
122 | if (defined($filename)) { |
---|
123 | &parse_config("$filename", \%opts) || |
---|
124 | die "Cannot read config file $filename: $!\n"; |
---|
125 | |
---|
126 | foreach my $opt (@expected_opts) { |
---|
127 | warn "Missing $opt option\n" if (!defined($opts{$opt})); |
---|
128 | } |
---|
129 | |
---|
130 | $active = 1 if ($opts{'active'} =~ /true/i); |
---|
131 | $tunnelcfg = 1 if ($opts{'tunnelcfg'} =~ /true/i); |
---|
132 | $type = $opts{'type'}; |
---|
133 | $type =~ tr/A-Z/a-z/; |
---|
134 | $remote_script_dir = $opts{'remotescriptdir'} || "."; |
---|
135 | $event_repeater = $opts{'eventrepeater'}; |
---|
136 | my $listen_on = $opts{'sshlisten'}; |
---|
137 | |
---|
138 | if (defined($opts{'fsname'})) { |
---|
139 | push(@ssh_port_fwds,"-R $listen_on:$SMBFS_PORT:$opts{'fsname'}:$SMBFS_PORT"); |
---|
140 | } |
---|
141 | |
---|
142 | if (defined($opts{'bossname'})) { |
---|
143 | push(@ssh_port_fwds, "-R $listen_on:$TMCD_PORT:$opts{'bossname'}:$TMCD_PORT"); |
---|
144 | } |
---|
145 | |
---|
146 | if (defined($opts{'eventservername'})) { |
---|
147 | push(@ssh_port_fwds,"-R ". |
---|
148 | "$listen_on:$remote_pubsub_port:$opts{'eventservername'}:$PUBSUB_PORT"); |
---|
149 | } |
---|
150 | if (defined($opts{'remoteeventservername'})) { |
---|
151 | push(@ssh_port_fwds,"-L :$remote_pubsub_port:" . |
---|
152 | "$opts{'remoteeventservername'}:$PUBSUB_PORT"); |
---|
153 | } |
---|
154 | |
---|
155 | # Forward connections to seer from remote TBs to control in this TB |
---|
156 | if (defined($opts{'seercontrol'})) { |
---|
157 | push(@ssh_port_fwds,"-R $listen_on:$SEER_PORT:$opts{seercontrol}:$SEER_PORT"); |
---|
158 | } |
---|
159 | |
---|
160 | # -n just starts the ssh tap tunnel |
---|
161 | @ssh_port_fwds = () if ($opts{'type'} eq 'experiment' || $opts{'n'}); |
---|
162 | |
---|
163 | print "ssh_port_fwds = ", join("\n",@ssh_port_fwds), "\n" if ($debug); |
---|
164 | } |
---|
165 | |
---|
166 | # Both sides need to have GatewayPorts and PermitTunnel set. Copy the existing |
---|
167 | # sshd_config, making sure GatewayPorts and PermitTunnel are set to yes, |
---|
168 | # replace the original, and restart sshd. |
---|
169 | my $ports_on = 0; |
---|
170 | my $tunnel_on = 0; |
---|
171 | |
---|
172 | my $conf = new IO::File($sshd_config) || die "Can't open $sshd_config: $!\n"; |
---|
173 | my $new_conf = new IO::File(">/tmp/sshd_config") || |
---|
174 | die "Can't open new ssh_config: $!\n"; |
---|
175 | |
---|
176 | while(<$conf>) { |
---|
177 | s/^\s*GatewayPorts.*/GatewayPorts yes/ && do { |
---|
178 | print $new_conf $_ unless $ports_on++; |
---|
179 | next; |
---|
180 | }; |
---|
181 | s/^\s*PermitTunnel.*/PermitTunnel yes/ && do { |
---|
182 | print $new_conf $_ unless $tunnel_on++; |
---|
183 | next; |
---|
184 | }; |
---|
185 | print $new_conf $_; |
---|
186 | } |
---|
187 | print $new_conf "GatewayPorts yes\n" unless $ports_on; |
---|
188 | print $new_conf "PermitTunnel yes\n" unless $tunnel_on; |
---|
189 | $conf->close(); |
---|
190 | $new_conf->close(); |
---|
191 | |
---|
192 | copy("/tmp/sshd_config", $sshd_config) || |
---|
193 | die "Cannot replace $sshd_config: $!\n"; |
---|
194 | |
---|
195 | system("/etc/rc.d/sshd restart"); |
---|
196 | |
---|
197 | # Need these to make the Ethernet tap and bridge to work... |
---|
198 | system("kldload /boot/kernel/bridgestp.ko") |
---|
199 | if -r "/boot/kernel/bridgestp.ko"; |
---|
200 | system("kldload /boot/kernel/if_bridge.ko"); |
---|
201 | system("kldload /boot/kernel/if_tap.ko"); |
---|
202 | |
---|
203 | if ($tunnelcfg && !$remote) { |
---|
204 | # Most Emulab-like testbeds use globally-routable addresses on the |
---|
205 | # control net; at DETER, we isolate the control net from the Internet. |
---|
206 | # On DETER-like testbeds, we need to create special tunneling nodes |
---|
207 | # with external access. Set up the external addresses as necessary. |
---|
208 | &setup_tunnel_cfg(%opts); |
---|
209 | } |
---|
210 | |
---|
211 | if (!$remote) { |
---|
212 | system("umask 077 && cp $opts{'privkeys'} /root/.ssh/id_rsa"); |
---|
213 | system("umask 077 && cp $opts{'pubkeys'} /root/.ssh/id_rsa.pub"); |
---|
214 | system("umask 077 && cat $opts{'pubkeys'} >> /root/.ssh/authorized_keys"); |
---|
215 | } |
---|
216 | |
---|
217 | if ($active) { |
---|
218 | # If we're the initiator, open up a separate tunnel to the remote |
---|
219 | # host for each of the different experiment net interfaces on this |
---|
220 | # machine. Execute this startup script on the far end, but with |
---|
221 | # the -r option to indicate that it's getting invoked remotely and |
---|
222 | # we should just handle the remote-end tasks. |
---|
223 | |
---|
224 | # Set up synchronization, so that the various user machines won't try to |
---|
225 | # contact boss before the tunnels are set up. (Thanks jjh!) |
---|
226 | |
---|
227 | do { |
---|
228 | system("$NC -z $opts{'peer'} $SSH_PORT"); |
---|
229 | } until (!$?); |
---|
230 | |
---|
231 | # XXX: Do we need to clear out previously created bridge interfaces? |
---|
232 | |
---|
233 | my $count = 0; |
---|
234 | my @SSHCMD; |
---|
235 | |
---|
236 | # If we are just setting up a control net connection, just fire up |
---|
237 | # ssh with the null command, and hang loose. |
---|
238 | |
---|
239 | if ($type eq "control") { |
---|
240 | foreach my $fwd (@ssh_port_fwds) { |
---|
241 | system("$SSH -N $fwd -o \"StrictHostKeyChecking no\" ". |
---|
242 | "$opts{'peer'} &"); #or die "Failed to run ssh"; |
---|
243 | } |
---|
244 | |
---|
245 | exit; |
---|
246 | } |
---|
247 | |
---|
248 | open(IFFILE, "/var/emulab/boot/ifmap") || die "couldn't open ifmap\n"; |
---|
249 | while (<IFFILE>) { |
---|
250 | my @a = split(' '); |
---|
251 | my $iface = $a[0]; |
---|
252 | my $addr = $a[1]; |
---|
253 | my $bridge = "bridge" . $count; |
---|
254 | my $tun = "tap" . $count; |
---|
255 | my $cmd; |
---|
256 | |
---|
257 | print "Found $iface, $addr, to bridge on $bridge\n" if ($debug); |
---|
258 | |
---|
259 | # Note that we're going to fire off an ssh which will never return; |
---|
260 | # we need the connection to stay up to keep the tunnel up. |
---|
261 | # In order to check for problems, we open it this way and read |
---|
262 | # the expected single line of output when the tunnel is connected. |
---|
263 | # To make debugging easier and to degrade more gracefully, I've split |
---|
264 | # these out into multiple processes. |
---|
265 | |
---|
266 | foreach my $fwd (@ssh_port_fwds) { |
---|
267 | $cmd = "$SSH -N $fwd -o \"StrictHostKeyChecking no\" ". |
---|
268 | "$opts{'peer'} &"; |
---|
269 | |
---|
270 | print "$cmd\n" if $debug; |
---|
271 | system("$cmd"); # or die "Failed to run ssh"; |
---|
272 | } |
---|
273 | # The Tunnel option specifies the level to tunnel at. Ethernet creates |
---|
274 | # a tap device rather than a tun device. Strict host key checking |
---|
275 | # avoids asking the user to OK a strange host key. |
---|
276 | $cmd = "$SSH -w $count:$count -o \"Tunnel ethernet\" " . |
---|
277 | "-o \"StrictHostKeyChecking no\" " . |
---|
278 | "$opts{'peer'} \"$remote_script_dir/fed-tun.pl " . |
---|
279 | "-r $addr $count\" & |"; |
---|
280 | |
---|
281 | print "$cmd\n" if $debug; |
---|
282 | |
---|
283 | open(SSHCMD, $cmd) |
---|
284 | or die "Failed to run ssh"; |
---|
285 | # Wait for the other end to report its work done. This string comes |
---|
286 | # from the block below (after the elsif) -- tvf |
---|
287 | my $check; # Make sure something ran... |
---|
288 | |
---|
289 | while ($check = <SSHCMD>) { |
---|
290 | last if $check =~ /^Remote connection all/; |
---|
291 | } |
---|
292 | print "Got line [$check] from remote side\n" if ($debug); |
---|
293 | |
---|
294 | &setup_bridging($tun, $bridge, $iface, $addr); |
---|
295 | $count++; |
---|
296 | @ssh_port_fwds = (); # only do this on the first connection |
---|
297 | } |
---|
298 | close(IFFILE); |
---|
299 | |
---|
300 | # Start a local event repeater (unless we're missing parameters |
---|
301 | die "Missing event repeater params (No config file ?)\n" |
---|
302 | unless $event_repeater && $opts{'remoteexperiment'} && |
---|
303 | $opts{'localexperiment'}; |
---|
304 | |
---|
305 | print "Starting event repeater\n" if $debug; |
---|
306 | |
---|
307 | print("$event_repeater -M -P $remote_pubsub_port -S localhost " . |
---|
308 | "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}\n") |
---|
309 | if $debug; |
---|
310 | # Connect to the forwarded pubsub port on this host to listen to the |
---|
311 | # other experiment's events, and to the local experiment to forward |
---|
312 | # events. |
---|
313 | system("$event_repeater -M -P $remote_pubsub_port -S localhost " . |
---|
314 | "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}"); |
---|
315 | warn "Event repeater returned $?\n" if $?; |
---|
316 | } elsif ($remote) { |
---|
317 | # We're on the remote system; figure out which interface to |
---|
318 | # tweak, based on the IP address passed in. |
---|
319 | |
---|
320 | my $iter = 0; |
---|
321 | my $iface; |
---|
322 | |
---|
323 | open(RTFILE, "$ROUTE_GET $addr |") || die "couldn't do $ROUTE_GET\n"; |
---|
324 | while (<RTFILE>) { |
---|
325 | if (/interface: (\w+)/) { |
---|
326 | $iface = $1; |
---|
327 | } |
---|
328 | } |
---|
329 | close(RTFILE); |
---|
330 | |
---|
331 | die "Couldn't find interface to use." if (!defined($iface)); |
---|
332 | my $bridge = "bridge" . $count; |
---|
333 | my $tun = "tap" . $count; |
---|
334 | |
---|
335 | &setup_bridging($tun, $bridge, $iface, $addr); |
---|
336 | print "Remote connection all set up!\n"; # Trigger other end with output |
---|
337 | |
---|
338 | # If this is the first remote invocation on a control gateway, start the |
---|
339 | # event repeater. |
---|
340 | |
---|
341 | if ( $count == 0 && $type ne "experiment" ) { |
---|
342 | my $remote_pubsub_port = $PUBSUB_PORT - 1; # There will be a local |
---|
343 | # pubsubd running, so we |
---|
344 | # dodge the port on the |
---|
345 | # remote tunnel node. |
---|
346 | # Make sure we have the relevant parameters |
---|
347 | die "Missing event repeater params (No config file ?)\n" |
---|
348 | unless $event_repeater && $opts{'remoteexperiment'} && |
---|
349 | $opts{'localexperiment'}; |
---|
350 | |
---|
351 | print "Starting event repeater\n" if $debug; |
---|
352 | |
---|
353 | # Connect to the forwarded pubsub port on this host to listen to the |
---|
354 | # other experiment's events, and to the local experiment to forward |
---|
355 | # events. |
---|
356 | system("$event_repeater -P $remote_pubsub_port -S localhost " . |
---|
357 | "-E $opts{'remoteexperiment'} -e $opts{'localexperiment'}"); |
---|
358 | warn "Event repeater returned $?\n" if $?; |
---|
359 | } |
---|
360 | } else { |
---|
361 | print "inactive end of a connection, finishing" if ($debug); |
---|
362 | } |
---|
363 | |
---|
364 | print "all done!\n" if ($debug); |
---|
365 | exit; |
---|
366 | |
---|
367 | |
---|
368 | # Set up the bridging for the new stuff... |
---|
369 | |
---|
370 | sub setup_bridging($; $; $; $) { |
---|
371 | my ($tun, $bridge, $iface, $addr) = @_; |
---|
372 | |
---|
373 | print "Waiting to see if new iface $tun is up\n" if ($debug); |
---|
374 | |
---|
375 | do { |
---|
376 | sleep 1; |
---|
377 | system("$IFCONFIG $tun"); |
---|
378 | } until (!$?); |
---|
379 | |
---|
380 | print "setting up $bridge with $iface and nuking $addr\n" if ($debug); |
---|
381 | |
---|
382 | system("ifconfig $bridge create"); |
---|
383 | system("ifconfig $iface delete $addr"); |
---|
384 | system("ifconfig $bridge addm $iface up"); |
---|
385 | system("ifconfig $bridge addm $tun"); |
---|
386 | } |
---|
387 | |
---|
388 | # Set up tunnel info for DETER-like testbeds. |
---|
389 | |
---|
390 | sub setup_tunnel_cfg { |
---|
391 | my (%opts) = @_; |
---|
392 | #my $tunnel_iface = "em0"; # XXX |
---|
393 | my $tunnel_iface = $opts{'interface'} || "em0"; |
---|
394 | my $tunnel_ip; |
---|
395 | my $tunnel_mask; |
---|
396 | my $tunnel_mac; |
---|
397 | my $tunnel_router; |
---|
398 | |
---|
399 | print "Opening $TMCC tunnelip\n" if ($debug); |
---|
400 | |
---|
401 | open(TMCD, "$TMCC tunnelip |") || die "tmcc failed\n"; |
---|
402 | print "Opened $TMCC tunnelip\n" if ($debug); |
---|
403 | while (<TMCD>) { |
---|
404 | print "got one line from tmcc\n" if ($debug); |
---|
405 | print if ($debug); |
---|
406 | if (/^TUNNELIP=([0-9.]*) TUNNELMASK=([0-9.]*) TUNNELMAC=(\w*) TUNNELROUTER=([0-9.]*)$/) { |
---|
407 | $tunnel_ip = $1; |
---|
408 | $tunnel_mask = $2; |
---|
409 | $tunnel_mac = $3; |
---|
410 | $tunnel_router = $4; |
---|
411 | } |
---|
412 | } |
---|
413 | close(TMCD); |
---|
414 | |
---|
415 | print "tunnel options: ip=$tunnel_ip mask=$tunnel_mask mac=$tunnel_mac router=$tunnel_router\n" if ($debug); |
---|
416 | |
---|
417 | if (defined($tunnel_mac) && ($tunnel_mac ne "")) { |
---|
418 | print "Opening $FINDIF $tunnel_mac\n" if ($debug); |
---|
419 | open(FIF, "$FINDIF $tunnel_mac |") || die "findif failed\n"; |
---|
420 | my $line = <FIF>; |
---|
421 | chomp($line); |
---|
422 | print "$FINDIF returned $line\n" if ($debug); |
---|
423 | $tunnel_iface = $line; |
---|
424 | } |
---|
425 | |
---|
426 | system("ifconfig $tunnel_iface $tunnel_ip" . |
---|
427 | ($tunnel_mask ? " netmask $tunnel_mask" : "") . " up"); |
---|
428 | warn "configuration of tunnel interface failed" if ($?); |
---|
429 | |
---|
430 | if ($tunnel_router) { |
---|
431 | |
---|
432 | # Sometimes the insertion of DNS names lags a bit. Retry this |
---|
433 | # configuration a few times to let DNS catch up. Might want to really |
---|
434 | # check the DNS name before we try this... |
---|
435 | my $config_succeeded = 0; |
---|
436 | my $tries = 0; |
---|
437 | my $max_retries = 300; |
---|
438 | |
---|
439 | do { |
---|
440 | system("route add $opts{'peer'} $tunnel_router"); |
---|
441 | if ( $? ) { |
---|
442 | warn "configuration routes via tunnel interface failed"; |
---|
443 | $tries++; |
---|
444 | sleep(10); |
---|
445 | } |
---|
446 | else { $config_succeeded = 1; } |
---|
447 | } until ( $config_succeeded || $tries > $max_retries ); |
---|
448 | } |
---|
449 | |
---|
450 | print "setup_tunnel_cfg done\n" if ($debug); |
---|
451 | } |
---|
452 | |
---|
453 | # Trick config-file parsing code from Ted Faber: |
---|
454 | |
---|
455 | # Parse the config file. The format is a colon-separated parameter name |
---|
456 | # followed by the value of that parameter to the end of the line. This parses |
---|
457 | # that format and puts the parameters into the referenced hash. Parameter |
---|
458 | # names are mapped to lower case, parameter values are unchanged. Returns 0 on |
---|
459 | # failure (e.g. file open) and 1 on success. |
---|
460 | sub parse_config { |
---|
461 | my($file, $href) = @_; |
---|
462 | my($fh) = new IO::File($file); |
---|
463 | |
---|
464 | unless ($fh) { |
---|
465 | warn "Can't open $file: $!\n"; |
---|
466 | return 0; |
---|
467 | } |
---|
468 | |
---|
469 | while (<$fh>) { |
---|
470 | next if /^\s*#/ || /^\s*$/; # Skip comments & blanks |
---|
471 | chomp; |
---|
472 | /^([^:]+):\s*(.*)/ && do { |
---|
473 | my($key) = $1; |
---|
474 | |
---|
475 | $key =~ tr/A-Z/a-z/; |
---|
476 | $href->{$key} = $2; |
---|
477 | next; |
---|
478 | }; |
---|
479 | warn "Unparasble line in $file: $_\n"; |
---|
480 | } |
---|
481 | $fh->close(); # It will close when it goes out of scope, but... |
---|
482 | return 1; |
---|
483 | } |
---|