Allow decoding files with invalid utf-8

src/_ubjson.c

@@ -26,7 +26,7 @@
 static _ubjson_encoder_prefs_t _ubjson_encoder_prefs_defaults = { NULL, 0, 0, 1 };
 
 // no_bytes, object_pairs_hook
-static _ubjson_decoder_prefs_t _ubjson_decoder_prefs_defaults = { NULL, NULL, 0, 0 };
+static _ubjson_decoder_prefs_t _ubjson_decoder_prefs_defaults = { NULL, NULL, 0, 0, NULL };
 
 /******************************************************************************/
 
@@ -98,8 +98,8 @@ PyDoc_STRVAR(_ubjson_load__doc__, "See pure Python version (encoder.load) for do
 #define FUNC_DEF_LOAD {"load", (PyCFunction)_ubjson_load, METH_VARARGS | METH_KEYWORDS, _ubjson_load__doc__}
 static PyObject*
 _ubjson_load(PyObject *self, PyObject *args, PyObject *kwargs) {
-    static const char *format = "O|iOOi:load";
-    static char *keywords[] = {"fp", "no_bytes", "object_hook", "object_pairs_hook", "intern_object_keys", NULL};
+    static const char *format = "O|iOOiz:load";
+    static char *keywords[] = {"fp", "no_bytes", "object_hook", "object_pairs_hook", "intern_object_keys", "errors", NULL};
 
     _ubjson_decoder_buffer_t *buffer = NULL;
     _ubjson_decoder_prefs_t prefs = _ubjson_decoder_prefs_defaults;
@@ -111,7 +111,7 @@ _ubjson_load(PyObject *self, PyObject *args, PyObject *kwargs) {
     UNUSED(self);
 
     if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, keywords, &fp, &prefs.no_bytes,  &prefs.object_hook,
-                                     &prefs.object_pairs_hook, &prefs.intern_object_keys)) {
+                                     &prefs.object_pairs_hook, &prefs.intern_object_keys, &prefs.errors)) {
         goto bail;
     }
 
@@ -153,8 +153,8 @@ PyDoc_STRVAR(_ubjson_loadb__doc__, "See pure Python version (encoder.loadb) for
 #define FUNC_DEF_LOADB {"loadb", (PyCFunction)_ubjson_loadb, METH_VARARGS | METH_KEYWORDS, _ubjson_loadb__doc__}
 static PyObject*
 _ubjson_loadb(PyObject *self, PyObject *args, PyObject *kwargs) {
-    static const char *format = "O|iOOi:loadb";
-    static char *keywords[] = {"chars", "no_bytes", "object_hook", "object_pairs_hook", "intern_object_keys", NULL};
+    static const char *format = "O|iOOiz:loadb";
+    static char *keywords[] = {"chars", "no_bytes", "object_hook", "object_pairs_hook", "intern_object_keys", "errors", NULL};
 
     _ubjson_decoder_buffer_t *buffer = NULL;
     _ubjson_decoder_prefs_t prefs = _ubjson_decoder_prefs_defaults;
@@ -163,7 +163,7 @@ _ubjson_loadb(PyObject *self, PyObject *args, PyObject *kwargs) {
     UNUSED(self);
 
     if (!PyArg_ParseTupleAndKeywords(args, kwargs, format, keywords, &chars, &prefs.no_bytes, &prefs.object_hook,
-                                     &prefs.object_pairs_hook, &prefs.intern_object_keys)) {
+                                     &prefs.object_pairs_hook, &prefs.intern_object_keys, &prefs.errors)) {
         goto bail;
     }
     if (PyUnicode_Check(chars)) {

src/decoder.c

@@ -85,8 +85,8 @@
     dst_char = tmp[0];\
 }
 
-#define DECODE_UNICODE_OR_BAIL(dst_obj, raw, length, item_str) {\
-    if (NULL == ((dst_obj) = PyUnicode_FromStringAndSize(raw, length))) {\
+#define DECODE_UNICODE_OR_BAIL(dst_obj, raw, length, item_str, errors) {\
+    if (NULL == ((dst_obj) = PyUnicode_DecodeUTF8(raw, length, errors))) {\
         RAISE_DECODER_EXCEPTION(("Failed to decode utf8: " item_str));\
     }\
 }\
@@ -567,7 +567,7 @@ static PyObject* _decode_high_prec(_ubjson_decoder_buffer_t *buffer) {
     DECODE_LENGTH_OR_BAIL(length);
     READ_OR_BAIL((Py_ssize_t)length, raw, "highprec");
 
-    DECODE_UNICODE_OR_BAIL(num_str, raw, (Py_ssize_t)length, "highprec");
+    DECODE_UNICODE_OR_BAIL(num_str, raw, (Py_ssize_t)length, "highprec", buffer->prefs.errors);
 
     BAIL_ON_NULL(decimal = PyObject_CallFunctionObjArgs((PyObject*)PyDec_Type, num_str, NULL));
     Py_XDECREF(num_str);
@@ -583,7 +583,7 @@ static PyObject* _decode_char(_ubjson_decoder_buffer_t *buffer) {
     PyObject *obj = NULL;
 
     READ_CHAR_OR_BAIL(value, "char");
-    DECODE_UNICODE_OR_BAIL(obj, &value, 1, "char");
+    DECODE_UNICODE_OR_BAIL(obj, &value, 1, "char", buffer->prefs.errors);
     return obj;
 
 bail:
@@ -600,7 +600,7 @@ static PyObject* _decode_string(_ubjson_decoder_buffer_t *buffer) {
 
     if (length > 0) {
         READ_OR_BAIL((Py_ssize_t)length, raw, "string");
-        DECODE_UNICODE_OR_BAIL(obj, raw, (Py_ssize_t)length, "string");
+        DECODE_UNICODE_OR_BAIL(obj, raw, (Py_ssize_t)length, "string", buffer->prefs.errors);
     } else {
         BAIL_ON_NULL(obj = PyUnicode_FromStringAndSize(NULL, 0));
     }

src/decoder.h

@@ -30,6 +30,7 @@ typedef struct {
     // don't convert UINT8 arrays to bytes instances (and keep as an array of individual integers)
     int no_bytes;
     int intern_object_keys;
+    char *errors;
 } _ubjson_decoder_prefs_t;
 
 typedef struct _ubjson_decoder_buffer_t {

test/test.py

@@ -134,6 +134,10 @@ def test_char(self):
         for suffix in (b'', b'\xfe'):
             with self.assertRaises(DecoderException):
                 self.ubjloadb(TYPE_CHAR + suffix)
+        # char invalid utf-8
+        for suffix in (b'\xfe',):
+            encoded = self.ubjloadb(TYPE_CHAR + suffix, errors='replace')
+            self.assertEqual(encoded, suffix.decode('utf-8', errors='replace'))
         for char in (u('a'), u('\0'), u('~')):
             self.check_enc_dec(char, 2)
 
@@ -144,6 +148,11 @@ def test_string(self):
         for suffix in (b'\x81', b'\x01', b'\x01' + b'\xfe'):
             with self.assertRaises(DecoderException):
                 self.ubjloadb(TYPE_STRING + TYPE_INT8 + suffix)
+        # string invalid utf-8
+        for length, suffix in ((b'\x01', b'\xfe'),):
+            suffix_d = suffix.decode('utf-8', errors='replace')
+            encoded = self.ubjloadb(TYPE_STRING + TYPE_INT8 + length + suffix, errors='replace')
+            self.assertEqual(encoded, suffix_d)
         # Note: In Python 2 plain str type is encoded as byte array
         for string in ('some ascii', u(r'\u00a9 with extended\u2122'), u('long string') * 100):
             self.check_enc_dec(string, 4, length_greater_or_equal=True)

ubjson/__init__.py

@@ -37,6 +37,6 @@
 from .encoder import EncoderException
 from .decoder import DecoderException
 
-__version__ = '0.16.1'
+__version__ = '0.16.2'
 
 __all__ = ('EXTENSION_ENABLED', 'dump', 'dumpb', 'EncoderException', 'load', 'loadb', 'DecoderException')