Changeset 1356 for pykota

Timestamp:

02/20/04 15:42:21 (21 years ago)

Author:

jalet

Message:

Experimental ldapcache directive added

Location:

pykota/trunk

Files:

• conf/pykota.conf.sample (modified) (1 diff)
• NEWS (modified) (1 diff)
• pykota/config.py (modified) (2 diffs)
• pykota/storages/ldapstorage.py (modified) (9 diffs)
• pykota/version.py (modified) (1 diff)

pykota/trunk/conf/pykota.conf.sample

@@ -118 +118 @@
 # common values are : memberUid, uniqueMember, member
 #groupmembers: memberUid
+
+# Activate low-level LDAP cache yes/no
+# Nothing to do with "storagecaching" which is higher level
+# and database independant.
+ldapcache: no
 
 # Where to log ?

pykota/trunk/NEWS

@@ -22 +22 @@
 PyKota NEWS :
 
+    - 1.18alpha7 :
+    
+        - New experimental ldapcache directive to improve
+          LDAP caching.
+          
     - 1.18alpha6 :
         

pykota/trunk/pykota/config.py

@@ -22 +22 @@
 #
 # $Log$
+# Revision 1.44  2004/02/20 14:42:21  jalet
+# Experimental ldapcache directive added
+#
 # Revision 1.43  2004/02/19 14:20:21  jalet
 # maildomain pykota.conf directive added.
@@ -471 +474 @@
         return self.isTrue(self.getGlobalOption("storagecaching", ignore=1))
             
+    def getLDAPCache(self) :          
+        """Returns 1 if low-level LDAP caching is enabled, else 0."""
+        return self.isTrue(self.getGlobalOption("ldapcache", ignore=1))
+            
     def getDisableHistory(self) :          
         """Returns 1 if we want to disable history, else 0."""

pykota/trunk/pykota/storages/ldapstorage.py

@@ -22 +22 @@
 #
 # $Log$
+# Revision 1.53  2004/02/20 14:42:21  jalet
+# Experimental ldapcache directive added
+#
 # Revision 1.52  2004/02/17 23:41:48  jalet
 # Preliminary work on low-level LDAP specific cache.
@@ -211 +214 @@
 try :
     import ldap
-    from ldap import modlist
+    import ldap.modlist
 except ImportError :    
     import sys
@@ -232 +235 @@
             raise PyKotaStorageError, "Unable to connect to LDAP server %s as %s." % (host, user) # TODO : translate
         else :    
-            self.ldapcache = {} # low-level cache specific to LDAP backend
+            self.useldapcache = pykotatool.config.getLDAPCache()
+            if self.useldapcache :
+                self.tool.logdebug("Low-Level LDAP Caching enabled.")
+                self.ldapcache = {} # low-level cache specific to LDAP backend
             self.closed = 0
             self.tool.logdebug("Database opened (host=%s, dbname=%s, user=%s)" % (host, dbname, user))
@@ -272 +278 @@
         self.tool.logdebug("Transaction aborted. WARNING : No transaction in LDAP !")
         
-    def doSearch(self, key, fields=None, base="", scope=ldap.SCOPE_SUBTREE) :
+    def doSearch(self, key, fields=None, base="", scope=ldap.SCOPE_SUBTREE, flushcache=0) :
         """Does an LDAP search query."""
         try :
             base = base or self.basedn
-            self.tool.logdebug("QUERY : Filter : %s, BaseDN : %s, Scope : %s, Attributes : %s" % (key, base, scope, fields))
-            result = self.database.search_s(base, scope, key, fields) # TODO : put 'None' instead of 'fields'
+            if self.useldapcache :
+                # Here we overwrite the fields the app want, to try and
+                # retrieve ALL user defined attributes ("*")
+                # + the createTimestamp attribute, needed by job history
+                # 
+                # This may not work with all LDAP servers
+                # but works at least in OpenLDAP (2.1.25) 
+                # and iPlanet Directory Server (5.1 SP3)
+                fields = ["*", "createTimestamp"]         
+                
+            if self.useldapcache and (not flushcache) and (scope == ldap.SCOPE_BASE) and self.ldapcache.has_key(base) :
+                entry = self.ldapcache[base]
+                self.tool.logdebug("LDAP cache hit %s => %s" % (base, entry))
+                result = [(base, entry)]
+            else :
+                self.tool.logdebug("QUERY : Filter : %s, BaseDN : %s, Scope : %s, Attributes : %s" % (key, base, scope, fields))
+                result = self.database.search_s(base, scope, key, fields)
         except ldap.LDAPError, msg :    
             raise PyKotaStorageError, (_("Search for %s(%s) from %s(scope=%s) returned no answer.") % (key, fields, base, scope)) + " : %s" % str(msg)
         else :     
             self.tool.logdebug("QUERY : Result : %s" % result)
-            for (dn, attributes) in result :
-                self.tool.logdebug("LDAP cache store (%s)" % dn)
-                self.ldapcache[dn] = attributes
+            if self.useldapcache :
+                for (dn, attributes) in result :
+                    self.tool.logdebug("LDAP cache store %s => %s" % (dn, attributes))
+                    self.ldapcache[dn] = attributes
             return result
             
@@ -293 +315 @@
         try :
             self.tool.logdebug("QUERY : ADD(%s, %s)" % (dn, str(fields)))
-            self.database.add_s(dn, modlist.addModlist(fields))
+            entry = ldap.modlist.addModlist(fields)
+            self.tool.logdebug("%s" % entry)
+            self.database.add_s(dn, entry)
         except ldap.LDAPError, msg :
             raise PyKotaStorageError, (_("Problem adding LDAP entry (%s, %s)") % (dn, str(fields))) + " : %s" % str(msg)
         else :
+            if self.useldapcache :
+                self.tool.logdebug("LDAP cache add %s => %s" % (dn, fields))
+                self.ldapcache[dn] = fields
             return dn
             
@@ -307 +334 @@
         except ldap.LDAPError, msg :
             raise PyKotaStorageError, (_("Problem deleting LDAP entry (%s)") % dn) + " : %s" % str(msg)
-            
-    def doModify(self, dn, fields, ignoreold=1) :
+        else :    
+            if self.useldapcache :
+                try :
+                    self.tool.logdebug("LDAP cache del %s" % dn)
+                    del self.ldapcache[dn]
+                except KeyError :    
+                    pass
+            
+    def doModify(self, dn, fields, ignoreold=1, flushcache=0) :
         """Modifies an entry in the LDAP directory."""
         try :
             # TODO : take care of, and update LDAP specific cache
-            if self.ldapcache.has_key(dn) :
-                self.tool.logdebug("LDAP cache hit (%s)" % dn)
-            else :    
-                self.tool.logdebug("LDAP cache miss (%s)" % dn)
-            oldentry = self.doSearch("objectClass=*", base=dn, scope=ldap.SCOPE_BASE)
+            if self.useldapcache and not flushcache :
+                if self.ldapcache.has_key(dn) :
+                    old = self.ldapcache[dn]
+                    self.tool.logdebug("LDAP cache hit %s => %s" % (dn, old))
+                    oldentry = {}
+                    for (k, v) in old.items() :
+                        if k != "createTimestamp" :
+                            oldentry[k] = v
+                else :    
+                    self.tool.logdebug("LDAP cache miss %s" % dn)
+                    oldentry = self.doSearch("objectClass=*", base=dn, scope=ldap.SCOPE_BASE)[0][1]
+            else :        
+                oldentry = self.doSearch("objectClass=*", base=dn, scope=ldap.SCOPE_BASE, flushcache=flushcache)[0][1]
             for (k, v) in fields.items() :
                 if type(v) == type({}) :
                     try :
-                        oldvalue = v["convert"](oldentry[0][1].get(k, [0])[0])
+                        oldvalue = v["convert"](oldentry.get(k, [0])[0])
                     except ValueError :    
-                        self.tool.logdebug("Error converting %s with %s(%s)" % (oldentry[0][1].get(k), k, v))
+                        self.tool.logdebug("Error converting %s with %s(%s)" % (oldentry.get(k), k, v))
                         oldvalue = 0
                     if v["operator"] == '+' :
@@ -330 +372 @@
                     fields[k] = str(newvalue)
             fields = self.normalizeFields(fields)
-            self.tool.logdebug("QUERY : Modify(%s, %s ==> %s)" % (dn, oldentry[0][1], fields))
-            self.database.modify_s(dn, modlist.modifyModlist(oldentry[0][1], fields, ignore_oldexistent=ignoreold))
+            self.tool.logdebug("QUERY : Modify(%s, %s ==> %s)" % (dn, oldentry, fields))
+            entry = ldap.modlist.modifyModlist(oldentry, fields, ignore_oldexistent=ignoreold)
+            self.database.modify_s(dn, entry)
         except ldap.LDAPError, msg :
             raise PyKotaStorageError, (_("Problem modifying LDAP entry (%s, %s)") % (dn, fields)) + " : %s" % str(msg)
         else :
+            if self.useldapcache :
+                cachedentry = self.ldapcache[dn]
+                for (mop, mtyp, mval) in entry :
+                    if mop in (ldap.MOD_ADD, ldap.MOD_REPLACE) :
+                        cachedentry[mtyp] = mval
+                    else :
+                        try :
+                            del cachedentry[mtyp]
+                        except KeyError :    
+                            pass
+                self.tool.logdebug("LDAP cache update %s => %s" % (dn, cachedentry))
             return dn
             
@@ -798 +852 @@
                    "pykotaBalance" : { "operator" : "-", "value" : amount, "convert" : float },
                  }
-        return self.doModify(user.idbalance, fields)         
+        return self.doModify(user.idbalance, fields, flushcache=1)         
        
     def writeUserAccountBalance(self, user, newbalance, newlifetimepaid=None) :    
@@ -807 +861 @@
         if newlifetimepaid is not None :
             fields.update({ "pykotaLifeTimePaid" : str(newlifetimepaid) })
-        return self.doModify(user.idbalance, fields)         
+        return self.doModify(user.idbalance, fields, flushcache=1)         
             
     def writeLastJobSize(self, lastjob, jobsize, jobprice) :        

pykota/trunk/pykota/version.py

@@ -22 +22 @@
 #
 
-__version__ = "1.18alpha6_unofficial"
+__version__ = "1.18alpha7_unofficial"
 
 __doc__ = """PyKota : a complete Printing Quota Solution for CUPS and LPRng."""