Merge pull request #93 from nowsecure/replacement_character

Use the unicode replacement char for non-printable characters

Author: bellini666

datagrid_gtk3/tests/utils/test_stringutils.py

@@ -7,7 +7,7 @@
 
 from datagrid_gtk3.utils.stringutils import (
     is_printable,
-    strip_non_printable,
+    replace_non_printable,
 )
 
 
@@ -30,15 +30,15 @@ def test_printable(self):
         for char in [chr(0), chr(20), chr(30)]:
             self.assertFalse(is_printable(char))
 
-    def test_strip_non_printable(self):
-        """Tests for :func:`datagrid.utils.stringutils.strip_non_printable`."""
+    def test_replace_non_printable(self):
+        """Test :func:`datagrid.utils.stringutils.replace_non_printable`."""
         self.assertEqual(
-            strip_non_printable(
+            replace_non_printable(
                 "Some string\nWith\tsome %s non-printable, %s chars" % (
                     chr(20), chr(30))),
-            "Some string\nWith\tsome  non-printable,  chars")
+            u"Some string\nWith\tsome � non-printable, � chars")
 
         self.assertEqual(
-            strip_non_printable(
+            replace_non_printable(
                 u"Ração %s para %s búfalos" % (chr(20), chr(30))),
-            u"Ração  para  búfalos")
+            u"Ração � para � búfalos")

datagrid_gtk3/utils/stringutils.py

@@ -14,11 +14,14 @@ def is_printable(char):
     return (char_code >= 32) or (9 <= char_code <= 13)
 
 
-def strip_non_printable(string_):
-    """Remove non-printable characters from the string.
+def replace_non_printable(string_):
+    """Replace non-printable characters on the string with a replacement.
 
-    :param string_: The string to remove the characters from
+    Use the unicode replacement character (U+FFFD), instead of the
+    non-printable ones.
+
+    :param string_: The string to replace the characters from
     :type string_: str
     :rtype: str
     """
-    return ''.join(c for c in string_ if is_printable(c))
+    return ''.join(c if is_printable(c) else u"\uFFFD" for c in string_)

datagrid_gtk3/utils/transformations.py

@@ -99,10 +99,9 @@ def string_transform(value, max_length=None, oneline=True,
                 raise
             value = decode_fallback(value)
 
-    # Remove non-printable characters from the string. Only keep those
-    # considered as whitespace/newline. We cannot use string.printable
-    # here as it would remove unicode characters too.
-    value = stringutils.strip_non_printable(value)
+    # Replace non-printable characters on the string so the user will
+    # know that there's something there even though it is not printable.
+    value = stringutils.replace_non_printable(value)
 
     if oneline:
         value = u' '.join(v.strip() for v in value.splitlines() if v.strip())