Changeset 0dc62df

Timestamp:

Nov 8, 2011 6:41:55 PM (13 years ago)

Author:

Ted Faber <faber@…>

Branches:

compt_changes, info-ops, master

Children:

db974ed

Parents:

8cb269a

Message:

Significantly improve resilience to SSL failures. #35

Location:

fedd/federation

Files:

• experiment_control.py (modified) (1 diff)
• remote_service.py (modified) (6 diffs)
• util.py (modified) (1 diff)

fedd/federation/experiment_control.py

@@ -88 +88 @@
     call_ReleaseAccess = service_caller('ReleaseAccess')
     call_StartSegment = service_caller('StartSegment')
-    call_TerminateSegment = service_caller('TerminateSegment')
+    call_TerminateSegment = service_caller('TerminateSegment', log = logging.getLogger("Test"))
     call_Ns2Topdl = service_caller('Ns2Topdl')
 

fedd/federation/remote_service.py

@@ -5 +5 @@
 from socket import error as socket_error
 from socket import sslerror
+from socket import SHUT_RDWR
 
 import M2Crypto.httpslib
@@ -417 +418 @@
         return self.apply_to_tags(resp, self.decap_fedids) 
 
+    def hammer_port_shut(self, port):
+        """
+        If there is an ssl failure, the connection to the service often stays
+        open and idle, confusing subsequent attempts to contact it (the symptom
+        is a stuck connection).  This routine walks through the open port and
+        hammers everything as closed as it can.  It is called a couple places
+        in call_soap_service.  This is dirty code, walking the internals of a
+        couple data structures, but without it, the daemon can lock up.
+        """
+        try:
+            if port is not None:
+                binding = getattr(port, 'binding', None)
+                if binding is not None: connection = getattr(binding, 'h', None)
+                else: connection = None
+
+                if connection is not None:
+                    csock = getattr(connection, 'sock', None)
+                    if csock is not None:
+                        csock.clear()
+                        csock.close()
+                        cssocket = getattr(csock, 'socket', None)
+                        if cssocket is not None:
+                            cssocket.shutdown(SHUT_RDWR)
+                            cssocket.close()
+                        del csock
+                        connection.sock = None
+                    httplib.HTTPConnection.close(connection)
+                    if self.log:
+                        self.log.debug("Closed connection with prejudice")
+
+                del port
+                port = None
+                if self.log:
+                    self.log.debug("Annihilated port")
+        except Exception, e:
+            port = None
+
+        return port
+
+
     def call_soap_service(self, url, req, cert_file=None, cert_pwd=None,
             trusted_certs=None, context=None, tracefile=None):
@@ -434 +475 @@
         ok = False
         retries = 0
+        port = None
         while not ok and retries < self.max_retries:
             try:
+                # If this is a retry, close the request and annihilate the port
+                port = self.hammer_port_shut(port)
+
                 # Reconstruct the full request message
                 msg = self.request_message()
@@ -487 +532 @@
                 ok = True
             except socket_error, e:
+                self.hammer_port_shut(port)
                 raise service_error(service_error.connect, 
                         "Cannot connect to %s: %s" % (url, e[1]))
@@ -510 +556 @@
                 retries +=1
             except ParseException, e:
+                port = self.hammer_port_shut(port)
                 raise service_error(service_error.protocol,
                         "Bad format message (XMLRPC??): %s" % e)
             except FaultException, e:
+                port = self.hammer_port_shut(port)
                 # If the method isn't implemented we get a FaultException
                 # without a detail (which would be a FeddFault).  If that's the
@@ -535 +583 @@
 
         if retries >= self.max_retries and fail_exc and not ok:
+            port = self.hammer_port_shut(port)
             raise service_error(service_error.connect, 
                     "Too many failures: %s" % fail_exc)

fedd/federation/util.py

@@ -106 +106 @@
                     callback=fedd_ssl_verify_callback)
 
+        # no session caching
+        self.set_session_cache_mode(0)
+
 class file_expanding_opts(OptionParser):
     def expand_file(self, option, opt_str, v, p):