Changeset dadc4da

Timestamp:

Mar 1, 2010 11:07:01 AM (15 years ago)

Author:

Ted Faber <faber@…>

Branches:

axis_example, compt_changes, info-ops, master, version-3.01, version-3.02

Children:

f52f5df

Parents:

9c2e4e1

Message:

revocation conrtol

File:

• fedd/federation/experiment_control.py (modified) (4 diffs)

fedd/federation/experiment_control.py

@@ -1075 +1075 @@
         # Wait until all finish (keep pinging the log, though)
         mins = 0
+        revoked = False
         while not thread_pool.wait_for_all_done(60.0):
             mins += 1
             alloc_log.info("Waiting for sub threads (it has been %d mins)" \
                     % mins)
+            if not revoked and \
+                    len([ t.getName() for t in threads if not t.rv ]) > 0:
+                # a testbed has failed.  Revoke this experiment's
+                # synchronizarion values so that sub experiments will not
+                # deadlock waiting for synchronization that will never happen
+                self.log.info("A subexperiment has failed to swap in, " + \
+                        "revoking synch keys")
+                var_key = "fedid:%s" % expid
+                for k in self.synch_store.all_keys():
+                    if len(k) > 45 and k[0:46] == var_key:
+                        self.synch_store.revoke_key(k)
+                revoked = True
 
         thread_pool.clear()
@@ -2659 +2672 @@
                     if len(k) > 45 and k[0:46] in fedid_keys:
                         self.synch_store.del_value(k)
-                except synch_store.BadDeleteionError:
+                except synch_store.BadDeletionError:
                     pass
             self.write_store()
@@ -2687 +2700 @@
 
         if self.auth.check_attribute(fid, name):
-            v = self.synch_store.get_value(name, wait)
+            try:
+                v = self.synch_store.get_value(name, wait)
+            except synch_store.RevokedKeyError:
+                # No more synch on this key
+                raise service_error(service_error.federant, 
+                        "Synch key %s revoked" % name)
             if v is not None:
                 rv['value'] = v
@@ -2717 +2735 @@
                 raise service_error(service_error.req,
                         "Value already set: %s" %name)
+            except synch_store.RevokedKeyError:
+                # No more synch on this key
+                raise service_error(service_error.federant, 
+                        "Synch key %s revoked" % name)
             return { 'name': name, 'value': v }
         else: