Changeset 1761

Timestamp:

10/02/04 07:48:56 (20 years ago)

Author:

jalet

Message:

Should now correctly deal with charsets both when storing into databases and when
retrieving datas. Works with both PostgreSQL and LDAP.

Location:

pykota/trunk

Files:

• cgi-bin/printquota.cgi (modified) (3 diffs)
• NEWS (modified) (1 diff)
• pykota/storages/ldapstorage.py (modified) (7 diffs)
• pykota/storages/sql.py (modified) (7 diffs)
• pykota/tool.py (modified) (4 diffs)
• pykota/version.py (modified) (1 diff)
• TODO (modified) (1 diff)

pykota/trunk/cgi-bin/printquota.cgi

@@ -24 +24 @@
 #
 # $Log$
+# Revision 1.37  2004/10/02 05:48:56  jalet
+# Should now correctly deal with charsets both when storing into databases and when
+# retrieving datas. Works with both PostgreSQL and LDAP.
+#
 # Revision 1.36  2004/09/02 10:34:09  jalet
 # Fixed problem with mod_auth_ldap Apache module
@@ -195 +199 @@
     langs = [l.strip().split(';')[0] for l in languages.split(",")]
     return "%s_%s" % (langs[0], langs[0].upper())
+    
+def getCharsetPreference() :
+    """Returns the preferred charset."""
+    charsets = os.environ.get("HTTP_ACCEPT_CHARSET", "no charset defined")
+    charsets = [l.strip().split(';')[0] for l in charsets.split(",")]
+    return charsets[0]
 
 class PyKotaReportGUI(PyKotaTool) :
@@ -351 +361 @@
 if __name__ == "__main__" :
     os.environ["LC_ALL"] = getLanguagePreference()
-    admin = PyKotaReportGUI(lang=os.environ["LC_ALL"])
+    admin = PyKotaReportGUI(lang=os.environ["LC_ALL"], charset=getCharsetPreference())
     admin.form = cgi.FieldStorage()
     admin.guiAction()

pykota/trunk/NEWS

@@ -22 +22 @@
 PyKota NEWS :
 
+    - 1.20alpha19 :
+    
+        - The description, filename, title, and options fields
+          are now stored as UTF-8 strings in both LDAP and PostgreSQL.
+          Proper charset is used during extraction from the database.
+          
     - 1.20alpha18 :
     

pykota/trunk/pykota/storages/ldapstorage.py

@@ -22 +22 @@
 #
 # $Log$
+# Revision 1.78  2004/10/02 05:48:56  jalet
+# Should now correctly deal with charsets both when storing into databases and when
+# retrieving datas. Works with both PostgreSQL and LDAP.
+#
 # Revision 1.77  2004/09/28 14:29:00  jalet
 # dumpykota for LDAP backend is almost there.
@@ -576 +580 @@
             printer.PricePerPage = float(fields.get("pykotaPricePerPage", [0.0])[0] or 0.0)
             printer.uniqueMember = fields.get("uniqueMember", [])
-            printer.Description = fields.get("description", [""])[0]
+            printer.Description = fields.get("description", [""])[0].decode("UTF-8").encode(self.tool.getCharset()) 
             printer.Exists = 1
         return printer    
@@ -677 +681 @@
                     lastjob.JobPrice = None
                 lastjob.JobAction = fields.get("pykotaAction", [""])[0]
-                lastjob.JobFileName = fields.get("pykotaFileName", [""])[0]
-                lastjob.JobTitle = fields.get("pykotaTitle", [""])[0]
+                lastjob.JobFileName = fields.get("pykotaFileName", [""])[0].decode("UTF-8").encode(self.tool.getCharset()) 
+                lastjob.JobTitle = fields.get("pykotaTitle", [""])[0].decode("UTF-8").encode(self.tool.getCharset()) 
                 lastjob.JobCopies = int(fields.get("pykotaCopies", [0])[0])
-                lastjob.JobOptions = fields.get("pykotaOptions", [""])[0]
+                lastjob.JobOptions = fields.get("pykotaOptions", [""])[0].decode("UTF-8").encode(self.tool.getCharset()) 
                 lastjob.JobHostName = fields.get("pykotaHostName", [""])[0]
                 lastjob.JobSizeBytes = fields.get("pykotaJobSizeBytes", [0L])[0]
@@ -753 +757 @@
                 printer.PricePerPage = float(fields.get("pykotaPricePerPage", [0.0])[0] or 0.0)
                 printer.uniqueMember = fields.get("uniqueMember", [])
-                printer.Description = fields.get("description", [""])[0]
+                printer.Description = fields.get("description", [""])[0].decode("UTF-8").encode(self.tool.getCharset()) 
                 printer.Exists = 1
                 printers.append(printer)
@@ -965 +969 @@
         """Write the printer's description back into the storage."""
         fields = {
-                   "description" : str(printer.Description),
+                   "description" : str(printer.Description).decode(self.tool.getCharset()).encode("UTF-8"), 
                  }
         self.doModify(printer.ident, fields)
@@ -1064 +1068 @@
                    "pykotaPrinterPageCounter" : str(pagecounter),
                    "pykotaAction" : action,
-                   "pykotaFileName" : ((filename is None) and "None") or filename.decode(os.environ.get("CHARSET", "ISO-8859-15")).encode("UTF-8"), 
-                   "pykotaTitle" : ((title is None) and "None") or title.decode(os.environ.get("CHARSET", "ISO-8859-15")).encode("UTF-8"), 
+                   "pykotaFileName" : ((filename is None) and "None") or filename.decode(self.tool.getCharset()).encode("UTF-8"), 
+                   "pykotaTitle" : ((title is None) and "None") or title.decode(self.tool.getCharset()).encode("UTF-8"), 
                    "pykotaCopies" : str(copies), 
-                   "pykotaOptions" : ((options is None) and "None") or options.decode(os.environ.get("CHARSET", "ISO-8859-15")).encode("UTF-8"), 
+                   "pykotaOptions" : ((options is None) and "None") or options.decode(self.tool.getCharset()).encode("UTF-8"), 
                    "pykotaHostName" : str(clienthost), 
                    "pykotaJobSizeBytes" : str(jobsizebytes),
@@ -1166 +1170 @@
                     job.JobPrice = None
                 job.JobAction = fields.get("pykotaAction", [""])[0]
-                job.JobFileName = fields.get("pykotaFileName", [""])[0]
-                job.JobTitle = fields.get("pykotaTitle", [""])[0]
+                job.JobFileName = fields.get("pykotaFileName", [""])[0].decode("UTF-8").encode(self.tool.getCharset()) 
+                job.JobTitle = fields.get("pykotaTitle", [""])[0].decode("UTF-8").encode(self.tool.getCharset()) 
                 job.JobCopies = int(fields.get("pykotaCopies", [0])[0])
-                job.JobOptions = fields.get("pykotaOptions", [""])[0]
+                job.JobOptions = fields.get("pykotaOptions", [""])[0].decode("UTF-8").encode(self.tool.getCharset()) 
                 job.JobHostName = fields.get("pykotaHostName", [""])[0]
                 job.JobSizeBytes = fields.get("pykotaJobSizeBytes", [0L])[0]

pykota/trunk/pykota/storages/sql.py

@@ -22 +22 @@
 #
 # $Log$
+# Revision 1.48  2004/10/02 05:48:56  jalet
+# Should now correctly deal with charsets both when storing into databases and when
+# retrieving datas. Works with both PostgreSQL and LDAP.
+#
 # Revision 1.47  2004/09/15 07:26:20  jalet
 # Data dumps are now ordered by entry creation date if applicable.
@@ -187 +191 @@
             printer.PricePerJob = fields.get("priceperjob") or 0.0
             printer.PricePerPage = fields.get("priceperpage") or 0.0
-            printer.Description = fields.get("description") or ""
+            printer.Description = (fields.get("description") or "").decode("UTF-8").encode(self.tool.getCharset()) 
             printer.Exists = 1
         return printer    
@@ -239 +243 @@
             lastjob.JobPrice = fields.get("jobprice")
             lastjob.JobAction = fields.get("action")
-            lastjob.JobFileName = fields.get("filename")
-            lastjob.JobTitle = fields.get("title")
+            lastjob.JobFileName = (fields.get("filename") or "").decode("UTF-8").encode(self.tool.getCharset()) 
+            lastjob.JobTitle = (fields.get("title") or "").decode("UTF-8").encode(self.tool.getCharset()) 
             lastjob.JobCopies = fields.get("copies")
-            lastjob.JobOptions = fields.get("options")
+            lastjob.JobOptions = (fields.get("options") or "").decode("UTF-8").encode(self.tool.getCharset()) 
             lastjob.JobDate = fields.get("jobdate")
             lastjob.JobHostName = fields.get("hostname")
@@ -301 +305 @@
                     printer.PricePerJob = record.get("priceperjob") or 0.0
                     printer.PricePerPage = record.get("priceperpage") or 0.0
-                    printer.Description = record.get("description") or ""
+                    printer.Description = (record.get("description") or "").decode("UTF-8").encode(self.tool.getCharset()) 
                     printer.Exists = 1
                     printers.append(printer)
@@ -387 +391 @@
     def writePrinterDescription(self, printer) :    
         """Write the printer's description back into the storage."""
-        self.doModify("UPDATE printers SET description=%s WHERE id=%s" % (self.doQuote(printer.Description), self.doQuote(printer.ident)))
+        description = printer.Description
+        if description is not None :
+            description = printer.Description.decode(self.tool.getCharset()).encode("UTF-8"), 
+        self.doModify("UPDATE printers SET description=%s WHERE id=%s" % (self.doQuote(description), self.doQuote(printer.ident)))
         
     def writeUserLimitBy(self, user, limitby) :    
@@ -432 +439 @@
         self.doModify("UPDATE jobhistory SET jobsize=%s, jobprice=%s WHERE id=%s" % (self.doQuote(jobsize), self.doQuote(jobprice), self.doQuote(lastjob.ident)))
         
-    def writeJobNew(self, printer, user, jobid, pagecounter, action, jobsize=None, jobprice=None, filename=None, title=None, copies=None, options=None, clienthost=None, jobsizebytes=None) :    
+    def writeJobNew(self, printer, user, jobid, pagecounter, action, jobsize=None, jobprice=None, filename=None, title=None, copies=None, options=None, clienthost=None, jobsizebytes=None) :
         """Adds a job in a printer's history."""
+        if filename is not None :
+            filename = filename.decode(self.tool.getCharset()).encode("UTF-8")
+        if title is not None :
+            title = title.decode(self.tool.getCharset()).encode("UTF-8")
+        if options is not None :
+            options = options.decode(self.tool.getCharset()).encode("UTF-8")
         if (not self.disablehistory) or (not printer.LastJob.Exists) :
             if jobsize is not None :
@@ -493 +506 @@
                 job.JobPrice = fields.get("jobprice")
                 job.JobAction = fields.get("action")
-                job.JobFileName = fields.get("filename")
-                job.JobTitle = fields.get("title")
+                job.JobFileName = (fields.get("filename") or "").decode("UTF-8").encode(self.tool.getCharset()) 
+                job.JobTitle = (fields.get("title") or "").decode("UTF-8").encode(self.tool.getCharset()) 
                 job.JobCopies = fields.get("copies")
-                job.JobOptions = fields.get("options")
+                job.JobOptions = (fields.get("options") or "").decode("UTF-8").encode(self.tool.getCharset()) 
                 job.JobDate = fields.get("jobdate")
                 job.JobHostName = fields.get("hostname")

pykota/trunk/pykota/tool.py

@@ -22 +22 @@
 #
 # $Log$
+# Revision 1.124  2004/10/02 05:48:56  jalet
+# Should now correctly deal with charsets both when storing into databases and when
+# retrieving datas. Works with both PostgreSQL and LDAP.
+#
 # Revision 1.123  2004/09/29 20:20:52  jalet
 # Added the winbind_separator directive to pykota.conf to allow the admin to
@@ -493 +497 @@
 class PyKotaTool :    
     """Base class for all PyKota command line tools."""
-    def __init__(self, lang=None, doc="PyKota %s (c) 2003-2004 %s" % (version.__version__, version.__author__)) :
+    def __init__(self, lang=None, charset=None, doc="PyKota %s (c) 2003-2004 %s" % (version.__version__, version.__author__)) :
         """Initializes the command line tool."""
         # locale stuff
@@ -501 +505 @@
         except (locale.Error, IOError) :
             gettext.NullTranslations().install()
+            
+        # We can force the charset.    
+        # The CHARSET environment variable is set by CUPS when printing.
+        # Else we use the current locale's one.
+        # If nothing is set, we use ISO-8859-15 widely used in western Europe.
+        self.charset = charset or os.environ.get("CHARSET") or locale.getlocale()[1] or locale.getdefaultlocale()[1] or "ISO-8859-15"
     
         # pykota specific stuff
@@ -522 +532 @@
         self.softwareJobSize = 0
         self.softwareJobPrice = 0.0
+        self.logdebug("Charset in use : %s" % self.charset)
+        
+    def getCharset(self) :    
+        """Returns the charset in use."""
+        return self.charset
         
     def logdebug(self, message) :    

pykota/trunk/pykota/version.py

@@ -22 +22 @@
 #
 
-__version__ = "1.20alpha18_unofficial"
+__version__ = "1.20alpha19_unofficial"
 
 __doc__ = """PyKota : a complete Printing Quota Solution for CUPS and LPRng."""

pykota/trunk/TODO

@@ -79 +79 @@
           pkusers --ingroups gr1,gr2,... --outgroups gr3,gr4,...
           
-        - Ensures data are stored in a consistent way with Unicode
-          under PostgreSQL : I can't reproduce the problem.
-        
         - Allow the --ingroups command line option to repykota, to
           report only users members of the specified groups.