Changeset 1130

Timestamp:

10/02/03 22:23:18 (21 years ago)

Author:

jalet

Message:

Storage caching mechanism added.

Location:

pykota/trunk

Files:

• conf/pykota.conf.sample (modified) (1 diff)
• NEWS (modified) (1 diff)
• pykota/config.py (modified) (2 diffs)
• pykota/storage.py (modified) (2 diffs)
• pykota/storages/ldapstorage.py (modified) (17 diffs)
• pykota/storages/pgstorage.py (modified) (14 diffs)
• pykota/tool.py (modified) (2 diffs)
• pykota/version.py (modified) (1 diff)

pykota/trunk/conf/pykota.conf.sample

@@ -51 +51 @@
 storageuser: pykotauser
 # storageuserpw: Comment out if unused, or set to Quota Storage user password
+
+# Should the database caching mechanism be enabled or not ?
+# If unset, caching is disabled. Possible values Y/N/YES/NO
+# caching mechanism works with both PostgreSQL and OpenLDAP backends
+# but may be interesting only with OpenLDAP.
+storagecaching: No
 
 # LDAP example, uncomment and adapt it to your own configuration :

pykota/trunk/NEWS

@@ -21 +21 @@
 
 PyKota NEWS :
+
+    - 1.15alpha5 :
+    
+        - Caching mechanism for all database queries.
 
     - 1.15alpha4 :

pykota/trunk/pykota/config.py

@@ -21 +21 @@
 #
 # $Log$
+# Revision 1.36  2003/10/02 20:23:18  jalet
+# Storage caching mechanism added.
+#
 # Revision 1.35  2003/07/29 09:54:03  jalet
 # Added configurable LDAP mail attribute support
@@ -401 +404 @@
         else :    
             return 0
+            
+    def getCaching(self) :          
+        """Returns 1 if database caching is enabled, else 0."""
+        caching = self.getGlobalOption("storagecaching", ignore=1)
+        if (caching is not None) and (caching.upper().strip() in ['Y', 'YES', '1', 'ON', 'O']) :
+            return 1
+        else :    
+            return 0

pykota/trunk/pykota/storage.py

@@ -21 +21 @@
 #
 # $Log$
+# Revision 1.20  2003/10/02 20:23:18  jalet
+# Storage caching mechanism added.
+#
 # Revision 1.19  2003/07/16 21:53:07  jalet
 # Really big modifications wrt new configuration file's location and content.
@@ -298 +301 @@
         self.JobSize = jobsize
     
+class BaseStorage :
+    def __init__(self, pykotatool) :
+        """Opens the LDAP connection."""
+        # raise PyKotaStorageError, "Sorry, the LDAP backend for PyKota is not yet implemented !"
+        self.closed = 1
+        self.tool = pykotatool
+        self.usecache = pykotatool.config.getCaching()
+        if self.usecache :
+            self.tool.logdebug("Caching enabled.")
+            self.caches = { "USERS" : {}, "GROUPS" : {}, "PRINTERS" : {}, "USERPQUOTAS" : {}, "GROUPPQUOTAS" : {}, "JOBS" : {}, "LASTJOBS" : {} }
+        
+    def __del__(self) :        
+        """Ensures that the database connection is closed."""
+        self.close()
+        
+    def getFromCache(self, cachetype, key) :
+        """Tries to extract something from the cache."""
+        if self.usecache :
+            entry = self.caches[cachetype].get(key)
+            if entry is not None :
+                self.tool.logdebug("Cache hit (%s->%s)" % (cachetype, key))
+            else :    
+                self.tool.logdebug("Cache miss (%s->%s)" % (cachetype, key))
+            return entry    
+            
+    def cacheEntry(self, cachetype, key, value) :        
+        """Puts an entry in the cache."""
+        if self.usecache :
+            self.caches[cachetype][key] = value
+            
+    def getUser(self, username) :        
+        """Returns the user from cache."""
+        user = self.getFromCache("USERS", username)
+        if user is None :
+            user = self.getUserFromBackend(username)
+            self.cacheEntry("USERS", username, user)
+        return user    
+        
+    def getGroup(self, groupname) :        
+        """Returns the group from cache."""
+        group = self.getFromCache("GROUPS", groupname)
+        if group is None :
+            group = self.getGroupFromBackend(groupname)
+            self.cacheEntry("GROUPS", groupname, group)
+        return group    
+        
+    def getPrinter(self, printername) :        
+        """Returns the printer from cache."""
+        printer = self.getFromCache("PRINTERS", printername)
+        if printer is None :
+            printer = self.getPrinterFromBackend(printername)
+            self.cacheEntry("PRINTERS", printername, printer)
+        return printer    
+        
+    def getUserPQuota(self, user, printer) :        
+        """Returns the user quota information from cache."""
+        useratprinter = "%s@%s" % (user.Name, printer.Name)
+        upquota = self.getFromCache("USERPQUOTAS", useratprinter)
+        if upquota is None :
+            upquota = self.getUserPQuotaFromBackend(user, printer)
+            self.cacheEntry("USERPQUOTAS", useratprinter, upquota)
+        return upquota    
+        
+    def getGroupPQuota(self, group, printer) :        
+        """Returns the group quota information from cache."""
+        groupatprinter = "%s@%s" % (group.Name, printer.Name)
+        gpquota = self.getFromCache("GROUPPQUOTAS", groupatprinter)
+        if gpquota is None :
+            gpquota = self.getGroupPQuotaFromBackend(group, printer)
+            self.cacheEntry("GROUPPQUOTAS", groupatprinter, gpquota)
+        return gpquota    
+        
+    def getPrinterLastJob(self, printer) :        
+        """Extracts last job information for a given printer from cache."""
+        lastjob = self.getFromCache("LASTJOBS", printer.Name)
+        if lastjob is None :
+            lastjob = self.getPrinterLastJobFromBackend(printer)
+            self.cacheEntry("LASTJOBS", printer.Name, lastjob)
+        return lastjob    
+        
 def openConnection(pykotatool) :
     """Returns a connection handle to the appropriate Quota Storage Database."""

pykota/trunk/pykota/storages/ldapstorage.py

@@ -21 +21 @@
 #
 # $Log$
+# Revision 1.26  2003/10/02 20:23:18  jalet
+# Storage caching mechanism added.
+#
 # Revision 1.25  2003/08/20 15:56:24  jalet
 # Better user and group deletion
@@ -116 +119 @@
 import md5
 
-from pykota.storage import PyKotaStorageError
-from pykota.storage import StorageObject,StorageUser,StorageGroup,StoragePrinter,StorageLastJob,StorageUserPQuota,StorageGroupPQuota
+from pykota.storage import PyKotaStorageError,BaseStorage,StorageObject,StorageUser,StorageGroup,StoragePrinter,StorageLastJob,StorageUserPQuota,StorageGroupPQuota
 
 try :
@@ -127 +129 @@
     raise PyKotaStorageError, "This python version (%s) doesn't seem to have the python-ldap module installed correctly." % sys.version.split()[0]
     
-class Storage :
+class Storage(BaseStorage) :
     def __init__(self, pykotatool, host, dbname, user, passwd) :
         """Opens the LDAP connection."""
         # raise PyKotaStorageError, "Sorry, the LDAP backend for PyKota is not yet implemented !"
-        self.closed = 1
-        self.tool = pykotatool
-        self.debug = pykotatool.config.getDebug()
+        BaseStorage.__init__(self, pykotatool)
         self.info = pykotatool.config.getLDAPInfo()
         try :
@@ -145 +145 @@
         else :    
             self.closed = 0
-            if self.debug :
-                self.tool.logger.log_message("Database opened (host=%s, dbname=%s, user=%s)" % (host, dbname, user), "debug")
-            
-    def __del__(self) :        
-        """Ensures that the database connection is closed."""
-        self.close()
-        
+            self.tool.logdebug("Database opened (host=%s, dbname=%s, user=%s)" % (host, dbname, user))
+            
     def close(self) :    
         """Closes the database connection."""
@@ -157 +152 @@
             del self.database
             self.closed = 1
-            if self.debug :
-                self.tool.logger.log_message("Database closed.", "debug")
+            self.tool.logdebug("Database closed.")
         
     def genUUID(self) :    
@@ -169 +163 @@
     def beginTransaction(self) :    
         """Starts a transaction."""
-        if self.debug :
-            self.tool.logger.log_message("Transaction begins... WARNING : No transactions in LDAP !", "debug")
+        self.tool.logdebug("Transaction begins... WARNING : No transactions in LDAP !")
         
     def commitTransaction(self) :    
         """Commits a transaction."""
-        if self.debug :
-            self.tool.logger.log_message("Transaction committed. WARNING : No transactions in LDAP !", "debug")
+        self.tool.logdebug("Transaction committed. WARNING : No transactions in LDAP !")
         
     def rollbackTransaction(self) :     
         """Rollbacks a transaction."""
-        if self.debug :
-            self.tool.logger.log_message("Transaction aborted. WARNING : No transaction in LDAP !", "debug")
+        self.tool.logdebug("Transaction aborted. WARNING : No transaction in LDAP !")
         
     def doSearch(self, key, fields=None, base="", scope=ldap.SCOPE_SUBTREE) :
@@ -186 +177 @@
         try :
             base = base or self.basedn
-            if self.debug :
-                self.tool.logger.log_message("QUERY : Filter : %s, BaseDN : %s, Scope : %s, Attributes : %s" % (key, base, scope, fields), "debug")
+            self.tool.logdebug("QUERY : Filter : %s, BaseDN : %s, Scope : %s, Attributes : %s" % (key, base, scope, fields))
             result = self.database.search_s(base or self.basedn, scope, key, fields)
         except ldap.LDAPError :    
             raise PyKotaStorageError, _("Search for %s(%s) from %s(scope=%s) returned no answer.") % (key, fields, base, scope)
         else :     
-            if self.debug :
-                self.tool.logger.log_message("QUERY : Result : %s" % result, "debug")
+            self.tool.logdebug("QUERY : Result : %s" % result)
             return result
             
@@ -199 +188 @@
         """Adds an entry in the LDAP directory."""
         try :
-            if self.debug :
-                self.tool.logger.log_message("QUERY : ADD(%s, %s)" % (dn, str(fields)), "debug")
+            self.tool.logdebug("QUERY : ADD(%s, %s)" % (dn, str(fields)))
             self.database.add_s(dn, modlist.addModlist(fields))
         except ldap.LDAPError :
@@ -210 +198 @@
         """Deletes an entry from the LDAP directory."""
         try :
-            if self.debug :
-                self.tool.logger.log_message("QUERY : Delete(%s)" % dn, "debug")
+            self.tool.logdebug("QUERY : Delete(%s)" % dn)
             self.database.delete_s(dn)
         except ldap.LDAPError :
@@ -220 +207 @@
         try :
             oldentry = self.doSearch("objectClass=*", base=dn, scope=ldap.SCOPE_BASE)
-            if self.debug :
-                self.tool.logger.log_message("QUERY : Modify(%s, %s ==> %s)" % (dn, oldentry[0][1], fields), "debug")
+            self.tool.logdebug("QUERY : Modify(%s, %s ==> %s)" % (dn, oldentry[0][1], fields))
             self.database.modify_s(dn, modlist.modifyModlist(oldentry[0][1], fields, ignore_oldexistent=ignoreold))
         except ldap.LDAPError :
@@ -228 +214 @@
             return dn
             
-    def getUser(self, username) :    
+    def getUserFromBackend(self, username) :    
         """Extracts user information given its name."""
         user = StorageUser(self, username)
@@ -262 +248 @@
         return user
        
-    def getGroup(self, groupname) :    
+    def getGroupFromBackend(self, groupname) :    
         """Extracts group information given its name."""
         group = StorageGroup(self, groupname)
@@ -282 +268 @@
         return group
        
-    def getPrinter(self, printername) :        
+    def getPrinterFromBackend(self, printername) :        
         """Extracts printer information given its name."""
         printer = StoragePrinter(self, printername)
@@ -294 +280 @@
             printer.Exists = 1
         return printer    
-            
-    def getUserGroups(self, user) :        
-        """Returns the user's groups list."""
-        groups = []
-        result = self.doSearch("(&(objectClass=pykotaGroup)(%s=%s))" % (self.info["groupmembers"], user.Name), [self.info["grouprdn"]], base=self.info["groupbase"])
-        if result :
-            for (groupid, fields) in result :
-                groups.append(self.getGroup(fields.get(self.info["grouprdn"])[0]))
-        return groups        
-        
-    def getGroupMembers(self, group) :        
-        """Returns the group's members list."""
-        groupmembers = []
-        result = self.doSearch("(&(objectClass=pykotaGroup)(|(pykotaGroupName=%s)(%s=%s)))" % (group.Name, self.info["grouprdn"], group.Name), [self.info["groupmembers"]], base=self.info["groupbase"])
-        if result :
-            for username in result[0][1].get(self.info["groupmembers"], []) :
-                groupmembers.append(self.getUser(username))
-        return groupmembers        
-        
-    def getUserPQuota(self, user, printer) :        
+        
+    def getUserPQuotaFromBackend(self, user, printer) :        
         """Extracts a user print quota."""
         userpquota = StorageUserPQuota(self, user, printer)
@@ -344 +312 @@
         return userpquota
         
-    def getGroupPQuota(self, group, printer) :        
+    def getGroupPQuotaFromBackend(self, group, printer) :        
         """Extracts a group print quota."""
         grouppquota = StorageGroupPQuota(self, group, printer)
@@ -383 +351 @@
         return grouppquota
         
-    def getPrinterLastJob(self, printer) :        
+    def getPrinterLastJobFromBackend(self, printer) :        
         """Extracts a printer's last job information."""
         lastjob = StorageLastJob(self, printer)
@@ -409 +377 @@
                 lastjob.Exists = 1
         return lastjob
+        
+    def getUserGroups(self, user) :        
+        """Returns the user's groups list."""
+        groups = []
+        result = self.doSearch("(&(objectClass=pykotaGroup)(%s=%s))" % (self.info["groupmembers"], user.Name), [self.info["grouprdn"]], base=self.info["groupbase"])
+        if result :
+            for (groupid, fields) in result :
+                groups.append(self.getGroup(fields.get(self.info["grouprdn"])[0]))
+        return groups        
+        
+    def getGroupMembers(self, group) :        
+        """Returns the group's members list."""
+        groupmembers = []
+        result = self.doSearch("(&(objectClass=pykotaGroup)(|(pykotaGroupName=%s)(%s=%s)))" % (group.Name, self.info["grouprdn"], group.Name), [self.info["groupmembers"]], base=self.info["groupbase"])
+        if result :
+            for username in result[0][1].get(self.info["groupmembers"], []) :
+                groupmembers.append(self.getUser(username))
+        return groupmembers        
         
     def getMatchingPrinters(self, printerpattern) :

pykota/trunk/pykota/storages/pgstorage.py

@@ -21 +21 @@
 #
 # $Log$
+# Revision 1.13  2003/10/02 20:23:18  jalet
+# Storage caching mechanism added.
+#
 # Revision 1.12  2003/08/17 14:20:25  jalet
 # Bug fix by Oleg Biteryakov
@@ -64 +67 @@
 #
 
-from pykota.storage import PyKotaStorageError
-from pykota.storage import StorageObject,StorageUser,StorageGroup,StoragePrinter,StorageLastJob,StorageUserPQuota,StorageGroupPQuota
+from pykota.storage import PyKotaStorageError,BaseStorage,StorageObject,StorageUser,StorageGroup,StoragePrinter,StorageLastJob,StorageUserPQuota,StorageGroupPQuota
 
 try :
@@ -74 +76 @@
     raise PyKotaStorageError, "This python version (%s) doesn't seem to have the PygreSQL module installed correctly." % sys.version.split()[0]
 
-class Storage :
+class Storage(BaseStorage) :
     def __init__(self, pykotatool, host, dbname, user, passwd) :
         """Opens the PostgreSQL database connection."""
-        self.tool = pykotatool
-        self.debug = pykotatool.config.getDebug()
-        self.closed = 1
+        BaseStorage.__init__(self, pykotatool)
         try :
             (host, port) = host.split(":")
@@ -92 +92 @@
         else :    
             self.closed = 0
-            if self.debug :
-                self.tool.logger.log_message("Database opened (host=%s, port=%s, dbname=%s, user=%s)" % (host, port, dbname, user), "debug")
-            
-    def __del__(self) :        
-        """Ensures that the database connection is closed."""
-        self.close()
-        
+            self.tool.logdebug("Database opened (host=%s, port=%s, dbname=%s, user=%s)" % (host, port, dbname, user))
+            
     def close(self) :    
         """Closes the database connection."""
@@ -104 +99 @@
             self.database.close()
             self.closed = 1
-            if self.debug :
-                self.tool.logger.log_message("Database closed.", "debug")
+            self.tool.logdebug("Database closed.")
         
     def beginTransaction(self) :    
         """Starts a transaction."""
         self.database.query("BEGIN;")
-        if self.debug :
-            self.tool.logger.log_message("Transaction begins...", "debug")
+        self.tool.logdebug("Transaction begins...")
         
     def commitTransaction(self) :    
         """Commits a transaction."""
         self.database.query("COMMIT;")
-        if self.debug :
-            self.tool.logger.log_message("Transaction committed.", "debug")
+        self.tool.logdebug("Transaction committed.")
         
     def rollbackTransaction(self) :     
         """Rollbacks a transaction."""
         self.database.query("ROLLBACK;")
-        if self.debug :
-            self.tool.logger.log_message("Transaction aborted.", "debug")
+        self.tool.logdebug("Transaction aborted.")
         
     def doSearch(self, query) :
@@ -131 +122 @@
             query += ';'
         try :
-            if self.debug :
-                self.tool.logger.log_message("QUERY : %s" % query, "debug")
+            self.tool.logdebug("QUERY : %s" % query)
             result = self.database.query(query)
         except pg.error, msg :    
@@ -146 +136 @@
             query += ';'
         try :
-            if self.debug :
-                self.tool.logger.log_message("QUERY : %s" % query, "debug")
+            self.tool.logdebug("QUERY : %s" % query)
             result = self.database.query(query)
         except pg.error, msg :    
@@ -164 +153 @@
         return pg._quote(field, typ)
         
-    def getUser(self, username) :    
+    def getUserFromBackend(self, username) :    
         """Extracts user information given its name."""
         user = StorageUser(self, username)
@@ -178 +167 @@
         return user
        
-    def getGroup(self, groupname) :    
+    def getGroupFromBackend(self, groupname) :    
         """Extracts group information given its name."""
         group = StorageGroup(self, groupname)
@@ -194 +183 @@
         return group
        
-    def getPrinter(self, printername) :        
+    def getPrinterFromBackend(self, printername) :        
         """Extracts printer information given its name."""
         printer = StoragePrinter(self, printername)
@@ -206 +195 @@
             printer.Exists = 1
         return printer    
-            
-    def getUserGroups(self, user) :        
-        """Returns the user's groups list."""
-        groups = []
-        result = self.doSearch("SELECT groupname FROM groupsmembers JOIN groups ON groupsmembers.groupid=groups.id WHERE userid=%s" % self.doQuote(user.ident))
-        if result :
-            for record in result :
-                groups.append(self.getGroup(record.get("groupname")))
-        return groups        
-        
-    def getGroupMembers(self, group) :        
-        """Returns the group's members list."""
-        groupmembers = []
-        result = self.doSearch("SELECT * FROM groupsmembers JOIN users ON groupsmembers.userid=users.id WHERE groupid=%s" % self.doQuote(group.ident))
-        if result :
-            for record in result :
-                user = StorageUser(self, record.get("username"))
-                user.ident = record.get("userid")
-                user.LimitBy = record.get("limitby")
-                user.AccountBalance = record.get("balance")
-                user.LifeTimePaid = record.get("lifetimepaid")
-                user.Email = record.get("email")
-                user.Exists = 1
-                groupmembers.append(user)
-        return groupmembers        
-        
-    def getUserPQuota(self, user, printer) :        
+        
+    def getUserPQuotaFromBackend(self, user, printer) :        
         """Extracts a user print quota."""
         userpquota = StorageUserPQuota(self, user, printer)
@@ -248 +212 @@
         return userpquota
         
-    def getGroupPQuota(self, group, printer) :        
+    def getGroupPQuotaFromBackend(self, group, printer) :        
         """Extracts a group print quota."""
         grouppquota = StorageGroupPQuota(self, group, printer)
@@ -267 +231 @@
         return grouppquota
         
-    def getPrinterLastJob(self, printer) :        
+    def getPrinterLastJobFromBackend(self, printer) :        
         """Extracts a printer's last job information."""
         lastjob = StorageLastJob(self, printer)
@@ -282 +246 @@
             lastjob.Exists = 1
         return lastjob
+            
+    def getUserGroups(self, user) :        
+        """Returns the user's groups list."""
+        groups = []
+        result = self.doSearch("SELECT groupname FROM groupsmembers JOIN groups ON groupsmembers.groupid=groups.id WHERE userid=%s" % self.doQuote(user.ident))
+        if result :
+            for record in result :
+                groups.append(self.getGroup(record.get("groupname")))
+        return groups        
+        
+    def getGroupMembers(self, group) :        
+        """Returns the group's members list."""
+        groupmembers = []
+        result = self.doSearch("SELECT * FROM groupsmembers JOIN users ON groupsmembers.userid=users.id WHERE groupid=%s" % self.doQuote(group.ident))
+        if result :
+            for record in result :
+                user = StorageUser(self, record.get("username"))
+                user.ident = record.get("userid")
+                user.LimitBy = record.get("limitby")
+                user.AccountBalance = record.get("balance")
+                user.LifeTimePaid = record.get("lifetimepaid")
+                user.Email = record.get("email")
+                user.Exists = 1
+                groupmembers.append(user)
+        return groupmembers        
         
     def getMatchingPrinters(self, printerpattern) :

pykota/trunk/pykota/tool.py

@@ -21 +21 @@
 #
 # $Log$
+# Revision 1.50  2003/10/02 20:23:18  jalet
+# Storage caching mechanism added.
+#
 # Revision 1.49  2003/07/29 20:55:17  jalet
 # 1.14 is out !
@@ -234 +237 @@
         self.config = config.PyKotaConfig("/etc/pykota")
         self.logger = logger.openLogger(self)
+        self.debug = self.config.getDebug()
         self.storage = storage.openConnection(self)
         self.smtpserver = self.config.getSMTPServer()
+        
+    def logdebug(self, message) :    
+        """Logs something to debug output if debug is enabled."""
+        if self.debug :
+            self.logger.log_message(message, "debug")
         
     def clean(self) :    

pykota/trunk/pykota/version.py

@@ -21 +21 @@
 #
 
-__version__ = "1.15alpha4_unofficial"
+__version__ = "1.15alpha5_unofficial"
 
 __doc__ = """PyKota : a complete Printing Quota Solution for CUPS and LPRng."""