fix: pos in JSONDecodeError (#169)

Author: PrettyWood

build.rs

@@ -5,7 +5,9 @@ fn main() {
         .args(&["-c", "import sys; print(sys.version_info[1])"])
         .output()
         .expect("python version did not print");
-    let version = String::from_utf8_lossy(&out.stdout).trim().parse::<u8>()
+    let version = String::from_utf8_lossy(&out.stdout)
+        .trim()
+        .parse::<u8>()
         .expect("python version was not parsed");
     for each in 6..version {
         println!("cargo:rustc-cfg=python3{}", each);

src/deserialize/deserializer.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 use crate::deserialize::cache::*;
+use crate::deserialize::DeserializeError;
 use crate::exc::*;
 use crate::ffi::*;
 use crate::typeref::*;
@@ -14,14 +15,14 @@ use std::ptr::NonNull;
 
 pub fn deserialize(
     ptr: *mut pyo3::ffi::PyObject,
-) -> std::result::Result<NonNull<pyo3::ffi::PyObject>, String> {
+) -> std::result::Result<NonNull<pyo3::ffi::PyObject>, DeserializeError<'static>> {
     let obj_type_ptr = ob_type!(ptr);
     let contents: &[u8];
     if is_type!(obj_type_ptr, STR_TYPE) {
         let mut str_size: pyo3::ffi::Py_ssize_t = 0;
         let uni = read_utf8_from_str(ptr, &mut str_size);
         if unlikely!(uni.is_null()) {
-            return Err(INVALID_STR.to_string());
+            return Err(DeserializeError::new(Cow::Borrowed(INVALID_STR), 0, 0, ""));
         }
         contents = unsafe { std::slice::from_raw_parts(uni, str_size as usize) };
     } else {
@@ -33,19 +34,29 @@ pub fn deserialize(
         } else if is_type!(obj_type_ptr, MEMORYVIEW_TYPE) {
             let membuf = unsafe { PyMemoryView_GET_BUFFER(ptr) };
             if unsafe { pyo3::ffi::PyBuffer_IsContiguous(membuf, b'C' as c_char) == 0 } {
-                return Err("Input type memoryview must be a C contiguous buffer".to_string());
+                return Err(DeserializeError::new(
+                    Cow::Borrowed("Input type memoryview must be a C contiguous buffer"),
+                    0,
+                    0,
+                    "",
+                ));
             }
             buffer = unsafe { (*membuf).buf as *const u8 };
             length = unsafe { (*membuf).len as usize };
         } else if is_type!(obj_type_ptr, BYTEARRAY_TYPE) {
             buffer = ffi!(PyByteArray_AsString(ptr)) as *const u8;
             length = ffi!(PyByteArray_Size(ptr)) as usize;
         } else {
-            return Err("Input must be bytes, bytearray, memoryview, or str".to_string());
+            return Err(DeserializeError::new(
+                Cow::Borrowed("Input must be bytes, bytearray, memoryview, or str"),
+                0,
+                0,
+                "",
+            ));
         }
         contents = unsafe { std::slice::from_raw_parts(buffer, length) };
         if encoding_rs::Encoding::utf8_valid_up_to(contents) != length {
-            return Err(INVALID_STR.to_string());
+            return Err(DeserializeError::new(Cow::Borrowed(INVALID_STR), 0, 0, ""));
         }
     }
 
@@ -55,10 +66,17 @@ pub fn deserialize(
     let seed = JsonValue {};
     match seed.deserialize(&mut deserializer) {
         Ok(obj) => {
-            deserializer.end().map_err(|e| e.to_string())?;
+            deserializer.end().map_err(|e| {
+                DeserializeError::new(Cow::Owned(e.to_string()), e.line(), e.column(), data)
+            })?;
             Ok(obj)
         }
-        Err(e) => Err(e.to_string()),
+        Err(e) => Err(DeserializeError::new(
+            Cow::Owned(e.to_string()),
+            e.line(),
+            e.column(),
+            data,
+        )),
     }
 }
 

src/deserialize/error.rs

@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+use std::borrow::Cow;
+
+#[derive(Debug, Clone)]
+pub struct DeserializeError<'a> {
+    pub message: Cow<'a, str>,
+    pub line: usize,   // start at 1
+    pub column: usize, // start at 1
+    pub data: &'a str,
+}
+
+impl<'a> DeserializeError<'a> {
+    #[cold]
+    pub fn new(message: Cow<'a, str>, line: usize, column: usize, data: &'a str) -> Self {
+        DeserializeError {
+            message,
+            line,
+            column,
+            data,
+        }
+    }
+
+    /// Return position of the error in the deserialized data
+    #[cold]
+    pub fn pos(&self) -> usize {
+        if self.line == 0 {
+            return 1;
+        }
+
+        self.data
+            .split('\n')
+            // take only the relevant lines
+            .take(self.line)
+            .enumerate()
+            .map(|(idx, s)| {
+                if idx == self.line - 1 {
+                    // Last line: only characters until the column of the error are relevant.
+                    // Note: Rust uses bytes whereas Python uses chars: we hence cannot
+                    //       directly use the `column` field
+                    if self.column == 0 { return 0; }
+
+                    let chars_count = s[..self.column - 1].chars().count();
+                    if chars_count == s.chars().count() - 1 {
+                        chars_count + 1
+                    } else {
+                        chars_count
+                    }
+                } else {
+                    // Other lines
+                    s.chars().count()
+                }
+            })
+            .sum::<usize>()
+            // add missed '\n' characters
+            + (self.line - 1)
+    }
+}

src/deserialize/mod.rs

@@ -2,7 +2,9 @@
 
 mod cache;
 mod deserializer;
+mod error;
 
 pub use cache::KeyMap;
 pub use cache::KEY_MAP;
 pub use deserializer::deserialize;
+pub use error::DeserializeError;

src/lib.rs

@@ -159,13 +159,16 @@ pub unsafe extern "C" fn PyInit_orjson() -> *mut PyObject {
 
 #[cold]
 #[inline(never)]
-fn raise_loads_exception(msg: Cow<str>) -> *mut PyObject {
+fn raise_loads_exception(err: deserialize::DeserializeError) -> *mut PyObject {
+    let pos = err.pos() as i64;
+    let msg = err.message;
+    let doc = err.data;
     unsafe {
         let err_msg =
             PyUnicode_FromStringAndSize(msg.as_ptr() as *const c_char, msg.len() as isize);
         let args = PyTuple_New(3);
-        let doc = PyUnicode_New(0, 255);
-        let pos = PyLong_FromLongLong(0);
+        let doc = PyUnicode_FromStringAndSize(doc.as_ptr() as *const c_char, doc.len() as isize);
+        let pos = PyLong_FromLongLong(pos);
         PyTuple_SET_ITEM(args, 0, err_msg);
         PyTuple_SET_ITEM(args, 1, doc);
         PyTuple_SET_ITEM(args, 2, pos);
@@ -191,7 +194,7 @@ fn raise_dumps_exception(msg: Cow<str>) -> *mut PyObject {
 pub unsafe extern "C" fn loads(_self: *mut PyObject, obj: *mut PyObject) -> *mut PyObject {
     match crate::deserialize::deserialize(obj) {
         Ok(val) => val.as_ptr(),
-        Err(err) => raise_loads_exception(Cow::Owned(err)),
+        Err(err) => raise_loads_exception(err),
     }
 }
 

test/test_error.py

@@ -0,0 +1,96 @@
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import json
+import unittest
+
+import pytest
+
+import orjson
+
+ASCII_TEST = b"""\
+{
+  "a": "qwe",
+  "b": "qweqwe",
+  "c": "qweq",
+  "d: "qwe"
+}
+"""
+
+MULTILINE_EMOJI = """[
+    "😊",
+    "a"
+"""
+
+
+class JsonDecodeErrorTests(unittest.TestCase):
+    def _get_error_infos(self, json_decode_error_exc_info):
+        return {
+            k: v
+            for k, v in json_decode_error_exc_info.value.__dict__.items()
+            if k in ("pos", "lineno", "colno")
+        }
+
+    def _test(self, data, expected_err_infos):
+        with pytest.raises(json.decoder.JSONDecodeError) as json_exc_info:
+            json.loads(data)
+
+        with pytest.raises(json.decoder.JSONDecodeError) as orjson_exc_info:
+            orjson.loads(data)
+
+        assert (
+            self._get_error_infos(json_exc_info)
+            == self._get_error_infos(orjson_exc_info)
+            == expected_err_infos
+        )
+
+    def test_ascii(self):
+        self._test(
+            ASCII_TEST,
+            {"pos": 55, "lineno": 5, "colno": 8},
+        )
+
+    def test_latin1(self):
+        self._test(
+            """["üýþÿ", "a" """,
+            {"pos": 13, "lineno": 1, "colno": 14},
+        )
+
+    def test_two_byte_str(self):
+        self._test(
+            """["東京", "a" """,
+            {"pos": 11, "lineno": 1, "colno": 12},
+        )
+
+    def test_two_byte_bytes(self):
+        self._test(
+            b'["\xe6\x9d\xb1\xe4\xba\xac", "a" ',
+            {"pos": 11, "lineno": 1, "colno": 12},
+        )
+
+    def test_four_byte(self):
+        self._test(
+            MULTILINE_EMOJI,
+            {"pos": 19, "lineno": 4, "colno": 1},
+        )
+
+    def test_tab(self):
+        # data/jsonchecker/fail26.json
+        data = """["tab\   character\   in\  string\  "]"""
+
+        with pytest.raises(json.decoder.JSONDecodeError) as json_exc_info:
+            json.loads(data)
+
+        assert self._get_error_infos(json_exc_info) == {
+            "pos": 5,
+            "lineno": 1,
+            "colno": 6,
+        }
+
+        with pytest.raises(json.decoder.JSONDecodeError) as orjson_exc_info:
+            orjson.loads(data)
+
+        assert self._get_error_infos(orjson_exc_info) == {
+            "pos": 6,
+            "lineno": 1,
+            "colno": 7,
+        }

test/test_numpy.py

@@ -444,6 +444,9 @@ def test_numpy_scalar_float64(self):
 
     def test_numpy_bool(self):
         self.assertEqual(
-            orjson.dumps({"a": numpy.bool_(True), "b": numpy.bool_(False)}, option=orjson.OPT_SERIALIZE_NUMPY),
-            b'{"a":true,"b":false}'
+            orjson.dumps(
+                {"a": numpy.bool_(True), "b": numpy.bool_(False)},
+                option=orjson.OPT_SERIALIZE_NUMPY,
+            ),
+            b'{"a":true,"b":false}',
         )