Merge remote-tracking branch 'anti-social/clean_xml'

Author: toastdriven

pysolr.py

@@ -181,6 +181,33 @@ def safe_urlencode(params, doseq=0):
     return urlencode(new_params, doseq)
 
 
+def is_valid_xml_char_ordinal(i):
+    """
+    Defines whether char is valid to use in xml document
+
+    XML standard defines a valid char as::
+
+    Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
+    """
+    return ( # conditions ordered by presumed frequency
+        0x20 <= i <= 0xD7FF
+        or i in (0x9, 0xA, 0xD)
+        or 0xE000 <= i <= 0xFFFD
+        or 0x10000 <= i <= 0x10FFFF
+        )
+
+
+def clean_xml_string(s):
+    """
+    Cleans string from invalid xml chars
+
+    Solution was found there::
+
+    http://stackoverflow.com/questions/8733233/filtering-out-certain-bytes-in-python
+    """
+    return ''.join(c for c in s if is_valid_xml_char_ordinal(ord(c)))
+
+
 class SolrError(Exception):
     pass
 
@@ -476,7 +503,7 @@ def _from_python(self, value):
 
             value = "{0}".format(value)
 
-        return value
+        return clean_xml_string(value)
 
     def _to_python(self, value):
         """

tests/client.py

@@ -4,7 +4,8 @@
 import datetime
 
 from pysolr import (Solr, Results, SolrError, unescape_html, safe_urlencode,
-                   force_unicode, force_bytes, sanitize, json, ET, IS_PY3)
+                    force_unicode, force_bytes, sanitize, json, ET, IS_PY3,
+                    clean_xml_string)
 
 try:
     import unittest2 as unittest
@@ -51,6 +52,9 @@ def test_force_bytes(self):
         # Don't mangle, it's already a bytestring.
         self.assertEqual(force_bytes(b'Hello \xe2\x98\x83'), b'Hello \xe2\x98\x83')
 
+    def test_clean_xml_string(self):
+        self.assertEqual(clean_xml_string('\x00\x0b\x0d\uffff'), '\x0d')
+
 
 class ResultsTestCase(unittest.TestCase):
     def test_init(self):
@@ -263,6 +267,7 @@ def test__from_python(self):
         self.assertEqual(self.solr._from_python(1.2), '1.2')
         self.assertEqual(self.solr._from_python(b'hello'), 'hello')
         self.assertEqual(self.solr._from_python('hello ☃'), 'hello ☃')
+        self.assertEqual(self.solr._from_python('\x01test\x02'), 'test')
 
     def test__to_python(self):
         self.assertEqual(self.solr._to_python('2013-01-18T00:00:00Z'), datetime.datetime(2013, 1, 18))