Changeset 3006 for pykota

Timestamp:

09/03/06 17:45:15 (18 years ago)

Author:

jerome

Message:

Now uses an automatic charset detection module if installed,
or fallback to the ISO-8859-15 charset, whenever we are in
a situation where the character set announced by CUPS (probably
always UTF-8) and the character set into which the title and
filenames are encoded don't match. This should "fix" the
annoying problem some people reported.

Location:

pykota/trunk

Files:

• checkdeps.py (modified) (1 diff)
• pykota/tool.py (modified) (4 diffs)
• README (modified) (1 diff)

pykota/trunk/checkdeps.py

@@ -82 +82 @@
                        ("Python-PAM", "PAM", "Python-PAM is recommended if you plan to use pknotify+PyKotIcon.\nGrab it from http://www.pangalactic.org/PyPAM/"),
                        ("Python-pkipplib", "pkipplib", "Python-pkipplib is now mandatory.\nGrab it from http://www.pykota.com/software/pkipplib/"),
+                       ("Python-chardet", "chardet", "Python-chardet is recommended.\nGrab it from http://chardet.feedparser.org/"),
                      ]
     commandstocheck = [ ("GhostScript", "gs", "Depending on your configuration, GhostScript may be needed in different parts of PyKota."),

pykota/trunk/pykota/tool.py

@@ -37 +37 @@
 
 from mx import DateTime
+
+try :
+    import chardet
+except ImportError :    
+    def detectCharset(text) :
+        """Fakes a charset detection if the chardet module is not installed."""
+        return "ISO-8859-15"
+else :    
+    def detectCharset(text) :
+        """Uses the chardet module to workaround CUPS lying to us."""
+        return chardet.detect(text)["encoding"]
 
 from pykota import config, storage, logger
@@ -133 +144 @@
         # Else we use the current locale's one.
         # If nothing is set, we use ISO-8859-15 widely used in western Europe.
-        localecharset = None
+        self.localecharset = None
         try :
             try :
-                localecharset = locale.nl_langinfo(locale.CODESET)
+                self.localecharset = locale.nl_langinfo(locale.CODESET)
             except AttributeError :    
                 try :
-                    localecharset = locale.getpreferredencoding()
+                    self.localecharset = locale.getpreferredencoding()
                 except AttributeError :    
                     try :
-                        localecharset = locale.getlocale()[1]
-                        localecharset = localecharset or locale.getdefaultlocale()[1]
+                        self.localecharset = locale.getlocale()[1]
+                        self.localecharset = self.localecharset or locale.getdefaultlocale()[1]
                     except ValueError :    
                         pass        # Unknown locale, strange...
         except locale.Error :            
             pass
-        self.charset = charset or os.environ.get("CHARSET") or localecharset or "ISO-8859-15"
+        self.charset = charset or os.environ.get("CHARSET") or self.localecharset or "ISO-8859-15"
     
         # pykota specific stuff
@@ -184 +195 @@
             self.printInfo("The 'pykota' system account is missing. Configuration files were searched in /etc/pykota instead.", "warn")
         
+        self.logdebug("Charset detected from locale settings : %s" % self.localecharset)
         self.logdebug("Charset in use : %s" % self.charset)
         arguments = " ".join(['"%s"' % arg for arg in sys.argv])
@@ -230 +242 @@
     def UTF8ToUserCharset(self, text) :
         """Converts from UTF-8 to user's charset."""
-        if text is not None :
+        if text is None :
+            return None
+        try :
+            return text.decode("UTF-8").encode(self.charset, "replace") 
+        except (UnicodeError, AttributeError) :    
             try :
-                return text.decode("UTF-8").encode(self.charset, "replace") 
+                # Maybe already in Unicode ?
+                return text.encode(self.charset, "replace") 
+            except (UnicodeError, AttributeError) :
+                # Try to autodetect the charset
+                return text.decode(detectCharset(text), "replace").encode(self.charset, "replace")
+        
+    def userCharsetToUTF8(self, text) :
+        """Converts from user's charset to UTF-8."""
+        if text is None :
+            return None
+        try :
+            # We don't necessarily trust the default charset, because
+            # xprint sends us titles in UTF-8 but CUPS gives us an ISO-8859-1 charset !
+            # So we first try to see if the text is already in UTF-8 or not, and
+            # if it is, we delete characters which can't be converted to the user's charset,
+            # then convert back to UTF-8. PostgreSQL 7.3.x used to reject some unicode characters,
+            # this is fixed by the ugly line below :
+            return text.decode("UTF-8").encode(self.charset, "replace").decode(self.charset).encode("UTF-8", "replace")
+        except (UnicodeError, AttributeError) :
+            try :
+                return text.decode(self.charset).encode("UTF-8", "replace") 
             except (UnicodeError, AttributeError) :    
                 try :
-                    # Maybe already in Unicode
-                    return text.encode(self.charset, "replace") 
+                    # Maybe already in Unicode ?
+                    return text.encode("UTF-8", "replace") 
                 except (UnicodeError, AttributeError) :
-                    pass # Don't know what to do
-        return text
-        
-    def userCharsetToUTF8(self, text) :
-        """Converts from user's charset to UTF-8."""
-        if text is not None :
-            try :
-                # We don't necessarily trust the default charset, because
-                # xprint sends us titles in UTF-8 but CUPS gives us an ISO-8859-1 charset !
-                # So we first try to see if the text is already in UTF-8 or not, and
-                # if it is, we delete characters which can't be converted to the user's charset,
-                # then convert back to UTF-8. PostgreSQL 7.3.x used to reject some unicode characters,
-                # this is fixed by the ugly line below :
-                return text.decode("UTF-8").encode(self.charset, "replace").decode(self.charset).encode("UTF-8", "replace")
-            except (UnicodeError, AttributeError) :
-                try :
-                    return text.decode(self.charset).encode("UTF-8", "replace") 
-                except (UnicodeError, AttributeError) :    
-                    try :
-                        # Maybe already in Unicode
-                        return text.encode("UTF-8", "replace") 
-                    except (UnicodeError, AttributeError) :
-                        pass # Don't know what to do
-        return text
+                    # Try to autodetect the charset
+                    return text.decode(detectCharset(text), "replace").encode("UTF-8", "replace")
+        return newtext
         
     def display(self, message) :

pykota/trunk/README

@@ -373 +373 @@
       (http://www.librelogiciel.com/software/)
     - The Python-PAM module if you need the pknotify command to be able  
-      to check usernames and passwords.
+      to check usernames and passwords. (http://www.pangalactic.org/PyPAM)
     - The Python-SNMP module to query printers for their page counter.
-      (http://pysnmp.sourceforge.net). 
+      (http://pysnmp.sourceforge.net) 
       IMPORTANT : version 3.4.2 or higher is REQUIRED.
       Versions 2.x won't work. Versions 4.x now work (tested with v4.1.5a).
     - The Python-OSD module to use the graphical print quota reminder.
       (http://repose.cx/pyosd/)
+    - The Python-chardet module to autodetect user's character set when  
+      printing. (http://chardet.feedparser.org)
     - SNMP tools (specifically the snmpget command) if you prefer to
       use your own script to request query printers.