Changeset 2867 for pykota/trunk

Timestamp:

04/14/06 00:13:03 (19 years ago)

Author:

jerome

Message:

Improved character encoding routines to handle incorrect input datas like
UTF-8 encoded strings received with an ISO-8859-1 charset, thanks to
CUPS and Xprint !

Location:

pykota/trunk

Files:

• NEWS (modified) (1 diff)
• pykota/tool.py (modified) (2 diffs)

pykota/trunk/NEWS

@@ -24 +24 @@
     - 1.25alpha1 (2006-04-10) :
     
-        - Fixed a non-ascii character encoding problem in the MySQL
-          backend code.
+        - Make the MySQL schema work with old releases of MySQL.
+        
+        - Fixed several character encoding problems.
           
         - Ensures that the denied banners counter is properly initialized

pykota/trunk/pykota/tool.py

@@ -231 +231 @@
         if text is not None :
             try :
-                return unicode(text, "UTF-8").encode(self.charset) 
-            except (UnicodeError, TypeError) :    
+                return text.decode("UTF-8").encode(self.charset, "replace") 
+            except (UnicodeError, AttributeError) :    
                 try :
-                    # Incorrect locale settings ?
-                    return unicode(text, "UTF-8").encode("ISO-8859-15") 
-                except (UnicodeError, TypeError) :    
-                    try :
-                        return text.encode(self.charset) 
-                    except (UnicodeError, TypeError, AttributeError) :
-                        pass
+                    # Maybe already in Unicode
+                    return text.encode(self.charset, "replace") 
+                except (UnicodeError, AttributeError) :
+                    pass # Don't know what to do
         return text
         
@@ -247 +244 @@
         if text is not None :
             try :
-                return unicode(text, self.charset).encode("UTF-8") 
-            except (UnicodeError, TypeError) :    
+                # We don't necessarily trust the default charset, because
+                # xprint sends us titles in UTF-8 but CUPS gives us an ISO-8859-1 charset !
+                # So we first try to see if the text is already in UTF-8 or not, and
+                # if it is, we delete characters which can't be converted to the user's charset,
+                # then convert back to UTF-8. PostgreSQL 7.3.x used to reject some unicode characters,
+                # this is fixed by the ugly line below :
+                return text.decode("UTF-8").encode(self.charset, "replace").decode(self.charset).encode("UTF-8", "replace")
+            except (UnicodeError, AttributeError) :
                 try :
-                    # Incorrect locale settings ?
-                    return unicode(text, "ISO-8859-15").encode("UTF-8") 
-                except (UnicodeError, TypeError) :    
+                    return text.decode(self.charset).encode("UTF-8", "replace") 
+                except (UnicodeError, AttributeError) :    
                     try :
-                        return text.encode("UTF-8") 
-                    except (UnicodeError, TypeError, AttributeError) :
-                        pass
+                        # Maybe already in Unicode
+                        return text.encode("UTF-8", "replace") 
+                    except (UnicodeError, AttributeError) :
+                        pass # Don't know what to do
         return text