Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cleanup code and port unicode module changes from orjson #160

Merged
merged 10 commits into from
Sep 29, 2023
Merged

Cleanup code and port unicode module changes from orjson #160

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
3d4beb6
Remove dead code
exg Sep 28, 2023
5a4fee1
Use pyo3-build-config #[cfg] flags
exg Sep 28, 2023
237c717
Use pyo3 types
exg Sep 28, 2023
a5efda5
Make ob_type macro work with type objects
exg Sep 28, 2023
4ebd497
Use is_type macro
exg Sep 28, 2023
d4a7f0f
Remove now unneeded unsafe blocks
exg Sep 28, 2023
3d846cc
Use std::ptr::null_mut
exg Sep 28, 2023
de9ceba
Port python-to-native string conversion changes from orjson
exg Sep 28, 2023
a586222
Port native-to-python string conversion changes from orjson
exg Sep 28, 2023
28f9791
Port python string hashing changes from orjson
exg Sep 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/deserialize/deserializer.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ impl<'de> Visitor<'de> for MsgpackValue {
},
);
pykey = entry.get();
pyhash = unsafe { (*pykey.cast::<PyASCIIObject>()).hash }
pyhash = unsafe { (*pykey.cast::<pyo3::ffi::PyASCIIObject>()).hash }
}
}
let _ = ffi!(_PyDict_SetItem_KnownHash(
Expand Down
17 changes: 0 additions & 17 deletions src/ffi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,29 +3,12 @@
use pyo3::ffi::*;
use std::os::raw::{c_char, c_int};

#[repr(C)]
#[derive(Debug, Copy, Clone)]
pub struct PyTypeObject {
pub ob_refcnt: pyo3::ffi::Py_ssize_t,
pub ob_type: *mut pyo3::ffi::PyTypeObject,
pub ma_used: pyo3::ffi::Py_ssize_t,
pub tp_name: *const c_char,
// ...
}

#[allow(non_snake_case)]
#[inline(always)]
pub unsafe fn PyDict_GET_SIZE(op: *mut PyObject) -> Py_ssize_t {
(*op.cast::<PyDictObject>()).ma_used
}

#[repr(C)]
pub struct PyBytesObject {
pub ob_base: PyVarObject,
pub ob_shash: Py_hash_t,
pub ob_sval: [c_char; 1],
}

#[allow(non_snake_case)]
#[inline(always)]
pub unsafe fn PyBytes_AS_STRING(op: *mut PyObject) -> *const c_char {
Expand Down
165 changes: 0 additions & 165 deletions src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ pub unsafe extern "C" fn PyInit_ormsgpack() -> *mut PyObject {
let wrapped_packb: PyMethodDef;
let wrapped_unpackb: PyMethodDef;

#[cfg(Py_3_8)]
{
wrapped_packb = PyMethodDef {
ml_name: "packb\0".as_ptr() as *const c_char,
Expand All @@ -96,17 +95,6 @@ pub unsafe extern "C" fn PyInit_ormsgpack() -> *mut PyObject {
};
}

#[cfg(not(Py_3_8))]
{
wrapped_packb = PyMethodDef {
ml_name: "packb\0".as_ptr() as *const c_char,
ml_meth: PyMethodDefPointer {
PyCFunctionWithKeywords: packb,
},
ml_flags: METH_VARARGS | METH_KEYWORDS,
ml_doc: PACKB_DOC.as_ptr() as *const c_char,
};
}
module_add!(
exported_objects,
"packb\0",
Expand All @@ -118,7 +106,6 @@ pub unsafe extern "C" fn PyInit_ormsgpack() -> *mut PyObject {
)
);

#[cfg(Py_3_8)]
{
wrapped_unpackb = PyMethodDef {
ml_name: "unpackb\0".as_ptr() as *const c_char,
Expand All @@ -130,18 +117,6 @@ pub unsafe extern "C" fn PyInit_ormsgpack() -> *mut PyObject {
};
}

#[cfg(not(Py_3_8))]
{
wrapped_unpackb = PyMethodDef {
ml_name: "unpackb\0".as_ptr() as *const c_char,
ml_meth: PyMethodDefPointer {
PyCFunctionWithKeywords: unpackb,
},
ml_flags: METH_VARARGS | METH_KEYWORDS,
ml_doc: UNPACKB_DOC.as_ptr() as *const c_char,
};
}

module_add!(
exported_objects,
"unpackb\0",
Expand Down Expand Up @@ -262,7 +237,6 @@ fn raise_packb_exception(msg: Cow<str>) -> *mut PyObject {
std::ptr::null_mut()
}

#[cfg(Py_3_8)]
#[no_mangle]
pub unsafe extern "C" fn unpackb(
_self: *mut PyObject,
Expand Down Expand Up @@ -320,70 +294,6 @@ pub unsafe extern "C" fn unpackb(
}
}

#[cfg(not(Py_3_8))]
#[no_mangle]
pub unsafe extern "C" fn unpackb(
_self: *mut PyObject,
args: *mut PyObject,
kwds: *mut PyObject,
) -> *mut PyObject {
let mut optsptr: Option<NonNull<PyObject>> = None;

let obj = PyTuple_GET_ITEM(args, 0);

let num_args = PyTuple_GET_SIZE(args);
if unlikely!(num_args != 1) {
let msg = if num_args > 1 {
Cow::Borrowed("unpackb() accepts only 1 positional argument")
} else {
Cow::Borrowed("unpackb() missing 1 required positional argument: 'obj'")
};
return raise_unpackb_exception(deserialize::DeserializeError::new(msg));
}

if !kwds.is_null() {
let len = unsafe { crate::ffi::PyDict_GET_SIZE(kwds) };
let mut pos = 0isize;
let mut arg: *mut PyObject = std::ptr::null_mut();
let mut val: *mut PyObject = std::ptr::null_mut();
for _ in 0..=len.saturating_sub(1) {
unsafe { _PyDict_Next(kwds, &mut pos, &mut arg, &mut val, std::ptr::null_mut()) };
if arg == typeref::OPTION {
optsptr = Some(NonNull::new_unchecked(val));
} else if arg.is_null() {
break;
} else {
return raise_unpackb_exception(deserialize::DeserializeError::new(Cow::Borrowed(
"unpackb() got an unexpected keyword argument",
)));
}
}
}

let mut optsbits: i32 = 0;
if let Some(opts) = optsptr {
let ob_type = (*opts.as_ptr()).ob_type;
if ob_type == typeref::INT_TYPE {
optsbits = PyLong_AsLong(optsptr.unwrap().as_ptr()) as i32;
if !(0..=opt::MAX_UNPACKB_OPT).contains(&optsbits) {
return raise_unpackb_exception(deserialize::DeserializeError::new(Cow::Borrowed(
"Invalid opts",
)));
}
} else if ob_type != typeref::NONE_TYPE {
return raise_unpackb_exception(deserialize::DeserializeError::new(Cow::Borrowed(
"Invalid opts",
)));
}
}

match crate::deserialize::deserialize(obj, optsbits as opt::Opt) {
Ok(val) => val.as_ptr(),
Err(err) => raise_unpackb_exception(err),
}
}

#[cfg(Py_3_8)]
#[no_mangle]
pub unsafe extern "C" fn packb(
_self: *mut PyObject,
Expand Down Expand Up @@ -452,78 +362,3 @@ pub unsafe extern "C" fn packb(
Err(err) => raise_packb_exception(Cow::Borrowed(&err)),
}
}

#[cfg(not(Py_3_8))]
#[no_mangle]
pub unsafe extern "C" fn packb(
_self: *mut PyObject,
args: *mut PyObject,
kwds: *mut PyObject,
) -> *mut PyObject {
let mut default: Option<NonNull<PyObject>> = None;
let mut optsptr: Option<NonNull<PyObject>> = None;

let obj = PyTuple_GET_ITEM(args, 0);

let num_args = PyTuple_GET_SIZE(args);
if unlikely!(num_args == 0) {
return raise_packb_exception(Cow::Borrowed(
"packb() missing 1 required positional argument: 'obj'",
));
}
if num_args & 2 == 2 {
default = Some(NonNull::new_unchecked(PyTuple_GET_ITEM(args, 1)));
}
if num_args & 3 == 3 {
optsptr = Some(NonNull::new_unchecked(PyTuple_GET_ITEM(args, 2)));
}

if !kwds.is_null() {
let len = unsafe { crate::ffi::PyDict_GET_SIZE(kwds) as usize };
let mut pos = 0isize;
let mut arg: *mut PyObject = std::ptr::null_mut();
let mut val: *mut PyObject = std::ptr::null_mut();
for _ in 0..=len - 1 {
unsafe { _PyDict_Next(kwds, &mut pos, &mut arg, &mut val, std::ptr::null_mut()) };
if arg == typeref::DEFAULT {
if unlikely!(num_args & 2 == 2) {
return raise_packb_exception(Cow::Borrowed(
"packb() got multiple values for argument: 'default'",
));
}
default = Some(NonNull::new_unchecked(val));
} else if arg == typeref::OPTION {
if unlikely!(num_args & 3 == 3) {
return raise_packb_exception(Cow::Borrowed(
"packb() got multiple values for argument: 'option'",
));
}
optsptr = Some(NonNull::new_unchecked(val));
} else if arg.is_null() {
break;
} else {
return raise_packb_exception(Cow::Borrowed(
"packb() got an unexpected keyword argument",
));
}
}
}

let mut optsbits: i32 = 0;
if let Some(opts) = optsptr {
let ob_type = (*opts.as_ptr()).ob_type;
if ob_type == typeref::INT_TYPE {
optsbits = PyLong_AsLong(optsptr.unwrap().as_ptr()) as i32;
if !(0..=opt::MAX_PACKB_OPT).contains(&optsbits) {
return raise_packb_exception(Cow::Borrowed("Invalid opts"));
}
} else if ob_type != typeref::NONE_TYPE {
return raise_packb_exception(Cow::Borrowed("Invalid opts"));
}
}

match crate::serialize::serialize(obj, default, optsbits as opt::Opt) {
Ok(val) => val.as_ptr(),
Err(err) => raise_packb_exception(Cow::Owned(err)),
}
}
18 changes: 8 additions & 10 deletions src/serialize/dataclass.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ impl Serialize for DataclassFastSerializer {
// length is unknown without further work because attributes are filtered below
let mut map = serializer.serialize_map(None).unwrap();
let mut pos = 0isize;
let mut str_size: pyo3::ffi::Py_ssize_t = 0;
let mut key: *mut pyo3::ffi::PyObject = std::ptr::null_mut();
let mut value: *mut pyo3::ffi::PyObject = std::ptr::null_mut();
for _ in 0..=len - 1 {
Expand All @@ -70,15 +69,15 @@ impl Serialize for DataclassFastSerializer {
self.recursion + 1,
self.default,
);
if unlikely!(unsafe { ob_type!(key) != STR_TYPE }) {
if unlikely!(!is_type!(ob_type!(key), STR_TYPE)) {
err!(KEY_MUST_BE_STR)
}
{
let data = read_utf8_from_str(key, &mut str_size);
if unlikely!(data.is_null()) {
let data = unicode_to_str(key);
if unlikely!(data.is_none()) {
err!(INVALID_STR)
}
let key_as_str = str_from_slice!(data, str_size);
let key_as_str = data.unwrap();
if unlikely!(key_as_str.as_bytes()[0] == b'_') {
continue;
}
Expand Down Expand Up @@ -132,7 +131,6 @@ impl Serialize for DataclassFallbackSerializer {
// length is unknown without further work because attributes are filtered below
let mut map = serializer.serialize_map(None).unwrap();
let mut pos = 0isize;
let mut str_size: pyo3::ffi::Py_ssize_t = 0;
let mut attr: *mut pyo3::ffi::PyObject = std::ptr::null_mut();
let mut field: *mut pyo3::ffi::PyObject = std::ptr::null_mut();
for _ in 0..=len - 1 {
Expand All @@ -147,15 +145,15 @@ impl Serialize for DataclassFallbackSerializer {
};
let field_type = ffi!(PyObject_GetAttr(field, FIELD_TYPE_STR));
ffi!(Py_DECREF(field_type));
if unsafe { field_type != FIELD_TYPE.as_ptr() } {
if !is_type!(field_type, FIELD_TYPE.as_ptr()) {
continue;
}
{
let data = read_utf8_from_str(attr, &mut str_size);
if unlikely!(data.is_null()) {
let data = unicode_to_str(attr);
if unlikely!(data.is_none()) {
err!(INVALID_STR);
}
let key_as_str = str_from_slice!(data, str_size);
let key_as_str = data.unwrap();
if key_as_str.as_bytes()[0] == b'_' {
continue;
}
Expand Down
37 changes: 13 additions & 24 deletions src/serialize/dict.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ impl Serialize for Dict {
let len = unsafe { PyDict_GET_SIZE(self.ptr) as usize };
let mut map = serializer.serialize_map(Some(len)).unwrap();
let mut pos = 0isize;
let mut str_size: pyo3::ffi::Py_ssize_t = 0;
let mut key: *mut pyo3::ffi::PyObject = std::ptr::null_mut();
let mut value: *mut pyo3::ffi::PyObject = std::ptr::null_mut();
for _ in 0..=len - 1 {
Expand All @@ -102,15 +101,15 @@ impl Serialize for Dict {
self.recursion + 1,
self.default,
);
if unlikely!(unsafe { ob_type!(key) != STR_TYPE }) {
if unlikely!(!is_type!(ob_type!(key), STR_TYPE)) {
err!(KEY_MUST_BE_STR)
}
{
let data = read_utf8_from_str(key, &mut str_size);
if unlikely!(data.is_null()) {
let data = unicode_to_str(key);
if unlikely!(data.is_none()) {
err!(INVALID_STR)
}
map.serialize_key(str_from_slice!(data, str_size)).unwrap();
map.serialize_key(data.unwrap()).unwrap();
}

map.serialize_value(&value)?;
Expand Down Expand Up @@ -215,14 +214,11 @@ impl DictNonStrKey {
}
ObType::Str => {
// because of ObType::Enum
let mut str_size: pyo3::ffi::Py_ssize_t = 0;
let uni = read_utf8_from_str(key, &mut str_size);
if unlikely!(uni.is_null()) {
let uni = unicode_to_str(key);
if unlikely!(uni.is_none()) {
Err(NonStrError::InvalidStr)
} else {
Ok(Key::String(InlinableString::from(str_from_slice!(
uni, str_size
))))
Ok(Key::String(InlinableString::from(uni.unwrap())))
}
}
ObType::Bytes => {
Expand All @@ -233,14 +229,11 @@ impl DictNonStrKey {
}))
}
ObType::StrSubclass => {
let mut str_size: pyo3::ffi::Py_ssize_t = 0;
let uni = ffi!(PyUnicode_AsUTF8AndSize(key, &mut str_size)) as *const u8;
if unlikely!(uni.is_null()) {
let uni = unicode_to_str_via_ffi(key);
if unlikely!(uni.is_none()) {
Err(NonStrError::InvalidStr)
} else {
Ok(Key::String(InlinableString::from(str_from_slice!(
uni, str_size
))))
Ok(Key::String(InlinableString::from(uni.unwrap())))
}
}
ObType::Tuple => Ok(Key::Tuple(key)),
Expand All @@ -265,7 +258,6 @@ impl Serialize for DictNonStrKey {
let mut items: SmallVec<[(Key, *mut pyo3::ffi::PyObject); 8]> =
SmallVec::with_capacity(len);
let mut pos = 0isize;
let mut str_size: pyo3::ffi::Py_ssize_t = 0;
let mut key: *mut pyo3::ffi::PyObject = std::ptr::null_mut();
let mut value: *mut pyo3::ffi::PyObject = std::ptr::null_mut();
let opts = self.opts & NOT_PASSTHROUGH;
Expand All @@ -280,14 +272,11 @@ impl Serialize for DictNonStrKey {
)
};
if is_type!(ob_type!(key), STR_TYPE) {
let data = read_utf8_from_str(key, &mut str_size);
if unlikely!(data.is_null()) {
let data = unicode_to_str(key);
if unlikely!(data.is_none()) {
err!(INVALID_STR)
}
items.push((
Key::String(InlinableString::from(str_from_slice!(data, str_size))),
value,
));
items.push((Key::String(InlinableString::from(data.unwrap())), value));
} else {
match DictNonStrKey::pyobject_to_string(key, opts) {
Ok(key_as_str) => items.push((key_as_str, value)),
Expand Down
Loading