Changeset 4abace972e38da89cefb4408c7ba1102baae23c1 for fedkit

Timestamp:

02/29/08 09:45:35 (4 years ago)

Author:

Ted Faber <faber@…>

Children:

f64fa816aea8218515cc0900940696d700329c0a

Parents:

906c76398e933601fe2e59e503084b2b645df5db

git-committer:

Ted Faber <faber@isi.edu> / 2008-02-29T17:45:35Z+0000

Message:

Changes to imporve reliability of routes coming up. Restructuring of
federate leaves the stsyem in a state where we can safely restart the router
(specifically, talking to the local boss again) and we restart it in the
final topology. This introduces a lag in routing actually being established.
It may be useful to add a lag to the startcmd to take this into account.

Location:

fedkit

Files:

• fed-tun.pl (modified) (1 diff)
• fed_bootstrap (added)
• federate.sh (modified) (5 diffs)
• splitter.pl (modified) (1 diff)
• testbeds.example (modified) (2 diffs)

fedkit/fed-tun.pl

@@ -265 +265 @@
            ($tunnel_mask ? " netmask $tunnel_mask" : ""));
     warn "configuration of tunnel interface failed" if ($?);
-        
-    system("route add $opts{'peer'} $tunnel_router");
-    warn "configuration routes via tunnel interface failed" if ($?);
+
+    # Sometimes the insertion of DNS names lags a bit.  Retry this
+    # configuration a few times to let DNS catch up.  Might want to really
+    # check the DNS name before we try this...
+    my $config_succeeded = 0;
+    my $tries = 0;
+    my $max_retries = 6;
+
+    do {
+        system("route add $opts{'peer'} $tunnel_router");
+        if ( $? ) {
+            warn "configuration routes via tunnel interface failed";
+            $tries++;
+            sleep(10);
+        }
+        else { $config_succeeded = 1; }
+    } until ( $config_succeeded || $tries > $max_retries );
 
     print "setup_tunnel_cfg done\n" if ($debug);

fedkit/federate.sh

@@ -1 +1 @@
 #!/bin/sh
 
-# Take the user and the user's startcmd as arguments
+# Set up a federated environment on a client.  Unmount all local file systems,
+# erase all local accounts and then bring in accounts and file systems from the
+# master testbed.  Once all that's done, restore client communication to the
+# local bossnode and execute any startup command passed in as arguments to this
+# script.
 
+
+# The file containing the DNS name of the current boss
 BOSSNODE="/usr/local/etc/emulab/bossnode"
-# GATEWAY="detertunnel"
 
+# Ports that the master gateway will forward
 PORTS="139 7777"
+# network cat command (to listen for the gateway)
 NC="/usr/bin/nc"
+# Emulab rc scripts directory
 RCDIR="/usr/local/etc/emulab/rc"
 
+# Emulab rc script to reset accounts and federation srcipt to start the smb
+# automount process.
 RCACCT="rc.accounts"
 SMBMOUNT="smbmount.pl"
@@ -26 +36 @@
 SCRIPTDIR=`$EMUDIR/tmcc -b status | perl -ne '/ALLOCATED=([^\/]+)\/([^\s]+)/ && print "/proj/$1/exp/$2/tmp\n";'`
 
-# Die if things are weird
+# Die if Scripts are not where they should be.  NB, this is checked by the
+# bootstrapper, too, so a failure here is unlikely.
 if [ -z "$SCRIPTDIR" ] ; then
     echo "Can't find federation scripts.  Tmcc status says:"
@@ -33 +44 @@
 fi
 
-# Move the SMB mounting script out of NFS so we can call it after the umount
-cp "$SCRIPTDIR/$SMBMOUNT" /tmp
-
 # Copy the global hostnames file into /etc/hosts so this node can address
-# others in the experiment on other testbeds.
+# others in the experiment on other testbeds.  The extra steps preserves teh
+# localhost entry that's aliased to this host's name.
 grep 127\\.0\\.0\\.1 /etc/hosts > /tmp/hosts
 cat "$SCRIPTDIR/hosts" >> /tmp/hosts
@@ -67 +76 @@
 
 echo "Configuring TMCC."
-
-$EMUDIR/sethostname
-
 echo $GATEWAY > $BOSSNODE
 
@@ -85 +91 @@
 
 #
-# Copy and exec the smbmount perl script
+# Call the smbmount perl script, this invokes the automounter to mount teh
+# needed smb filesystems.
 #
-
 echo "Mounting via SMB."
-
 "/tmp/$SMBMOUNT" $SHARE $GATEWAY $SMBUSER 
 
+echo "Restoring old bossnode"
+# remove bossnode override
+/bin/rm -f $BOSSNODE
+# Also clear any cached data that might interfere with rourte construction.
+/bin/rm -f /var/emulab/boot/tmcc/ifconfig
+
+# I remain unclear why ospfd seems screwed up after the resetting of accounts
+# and mounts.  This seems to fix it, but it's not very satisfying.  At this
+# point the federated topology is complete, so any problems with routing in a
+# partial topology -e.g. failed or partial bridging - that might have confused
+# the router should be gone.
+/bin/pkill ospf
+/bin/pkill gated
+/usr/local/etc/emulab/rc/rc.route boot
+
+# Execute any command passed in as a startcmd.  This basically daisy chains the
+# startcmd.
 if [ ! -z "$1" ]; then 
         su -l "$1" -c "$2"

fedkit/splitter.pl

@@ -9 +9 @@
 use File::Copy;
 
-my @scripts = ("federate.sh", "smbmount.pl", "make_hosts", "fed-tun.pl");
+my @scripts = ("fed_bootstrap", "federate.sh", "smbmount.pl", 
+    "make_hosts", "fed-tun.pl");
 my $local_script_dir = ".";
 my($pid, $gid);                 # Process and group IDs for calling parse.tcl

fedkit/testbeds.example

@@ -49 +49 @@
 # These slave commands are appropriate for generic testbeds, which emulab and
 # WAIL are.  The -H on sudo is crucial when calling federate.sh
-SlaveNodeStartCmd: sudo -H /bin/sh FEDDIR/federate.sh >& /tmp/federate
+SlaveNodeStartCmd: sudo -H /bin/sh FEDDIR/fed_bootstrap >& /tmp/federate
 SlaveConnectorStartCmd: sudo -H FEDDIR/fed-tun.pl -f GWCONF>& /tmp/bridge.log
 
 # These master commands are placeholders
-MasterNodeStartCmd: sudo -H /bin/sh FEDDIR/federate.sh &> /tmp/mfederate 
+MasterNodeStartCmd: sudo -H /bin/sh FEDDIR/fed_bootstrap &> /tmp/mfederate 
 MasterConnectorStartCmd: sudo -H /users/faber/fed-tun.pl >& /tmp/bridge.log
 FileServer: fs
@@ -66 +66 @@
 ConnectorType: pc
 ConnectorImage: FBSD61-TUNNEL2
-SlaveNodeStartCmd: sudo -H /bin/sh FEDDIR/federate.sh >& /tmp/federate
+SlaveNodeStartCmd: sudo -H /bin/sh FEDDIR/fed_bootstrap >& /tmp/federate
 SlaveConnectorStartCmd: sudo -H FEDDIR/fed-tun.pl -f GWCONF>& /tmp/bridge.log
-MasterNodeStartCmd: sudo -H /bin/sh FEDDIR/federate.sh &> /tmp/mfederate 
+MasterNodeStartCmd: sudo -H /bin/sh FEDDIR/fed_bootstrap &> /tmp/mfederate 
 MasterConnectorStartCmd: sudo -H /users/faber/fed-tun.pl >& /tmp/bridge.log
 FileServer: fs