From 4b1d0c4c99c2d2607c1bb8405a2c45e007212271 Mon Sep 17 00:00:00 2001 From: Carl Kadie Date: Sat, 23 Dec 2023 18:57:34 -0800 Subject: [PATCH] remove unsafe --- bed_reader/_open_bed_cloud.py | 28 +++---- bed_reader/tests/test_open_bed.py | 14 ++-- bed_reader/tests/test_open_bed_cloud.py | 98 ++++++++++++------------- src/python_module.rs | 78 +++++++++++++------- 4 files changed, 123 insertions(+), 95 deletions(-) diff --git a/bed_reader/_open_bed_cloud.py b/bed_reader/_open_bed_cloud.py index d33deb8..d56b396 100644 --- a/bed_reader/_open_bed_cloud.py +++ b/bed_reader/_open_bed_cloud.py @@ -566,8 +566,8 @@ async def iid(self) -> np.ndarray: """ return self.property_item("iid") - @property - def father(self) -> np.ndarray: + # @property + async def father(self) -> np.ndarray: """ Father id of each individual (sample). @@ -596,8 +596,8 @@ def father(self) -> np.ndarray: """ return self.property_item("father") - @property - def mother(self) -> np.ndarray: + # @property + async def mother(self) -> np.ndarray: """ Mother id of each individual (sample). @@ -626,8 +626,8 @@ def mother(self) -> np.ndarray: """ return self.property_item("mother") - @property - def sex(self) -> np.ndarray: + # @property + async def sex(self) -> np.ndarray: """ Sex of each individual (sample). @@ -655,8 +655,8 @@ def sex(self) -> np.ndarray: """ return self.property_item("sex") - @property - def pheno(self) -> np.ndarray: + # @property + async def pheno(self) -> np.ndarray: """ A phenotype for each individual (sample) (seldom used). @@ -820,8 +820,8 @@ async def sid(self) -> np.ndarray: """ return self.property_item("sid") - @property - def cm_position(self) -> np.ndarray: + # @property + async def cm_position(self) -> np.ndarray: """ Centimorgan position of each SNP (variant). @@ -880,8 +880,8 @@ async def bp_position(self) -> np.ndarray: """ return self.property_item("bp_position") - @property - def allele_1(self) -> np.ndarray: + # @property + async def allele_1(self) -> np.ndarray: """ First allele of each SNP (variant). @@ -908,8 +908,8 @@ def allele_1(self) -> np.ndarray: """ return self.property_item("allele_1") - @property - def allele_2(self) -> np.ndarray: + # @property + async def allele_2(self) -> np.ndarray: """ Second allele of each SNP (variant), diff --git a/bed_reader/tests/test_open_bed.py b/bed_reader/tests/test_open_bed.py index 8576664..ff281ce 100644 --- a/bed_reader/tests/test_open_bed.py +++ b/bed_reader/tests/test_open_bed.py @@ -46,13 +46,13 @@ def test_write(tmp_path, shared_datadir): assert np.allclose(val0, bed1.read(), equal_nan=True) val_sparse = bed1.read_sparse() assert np.allclose(val0, val_sparse.toarray(), equal_nan=True) - assert np.array_equal(bed.fid, properties0["fid"]) - assert np.array_equal(bed.iid, properties0["iid"]) - assert np.array_equal(bed.sid, properties0["sid"]) - assert np.issubdtype(bed.sid.dtype, np.str_) - assert np.array_equal(bed.chromosome, properties0["chromosome"]) - assert np.allclose(bed.cm_position, properties0["cm_position"]) - assert np.allclose(bed.bp_position, properties0["bp_position"]) + assert np.array_equal(bed1.fid, properties0["fid"]) + assert np.array_equal(bed1.iid, properties0["iid"]) + assert np.array_equal(bed1.sid, properties0["sid"]) + assert np.issubdtype(bed1.sid.dtype, np.str_) + assert np.array_equal(bed1.chromosome, properties0["chromosome"]) + assert np.allclose(bed1.cm_position, properties0["cm_position"]) + assert np.allclose(bed1.bp_position, properties0["bp_position"]) val_float = val0.astype("float") val_float[0, 0] = 0.5 diff --git a/bed_reader/tests/test_open_bed_cloud.py b/bed_reader/tests/test_open_bed_cloud.py index 53d4fa0..878f2bb 100644 --- a/bed_reader/tests/test_open_bed_cloud.py +++ b/bed_reader/tests/test_open_bed_cloud.py @@ -6,7 +6,7 @@ import numpy as np import pytest -from bed_reader import open_bed_cloud +from bed_reader import open_bed_cloud, to_bed @pytest.mark.asyncio @@ -28,54 +28,54 @@ async def test_cloud_read1(shared_datadir): assert (await bed_cloud.bp_position())[-1] == 100 -# def test_cloud_write(tmp_path, shared_datadir): -# in_file = shared_datadir / "plink_sim_10s_100v_10pmiss.bed" -# out_file = tmp_path / "out.bed" -# with open_bed_cloud(in_file) as bed: -# val0 = bed.read() -# properties0 = { -# "fid": bed.fid, -# "iid": bed.iid, -# "sid": bed.sid, -# "chromosome": bed.chromosome, -# "cm_position": bed.cm_position, -# "bp_position": bed.bp_position, -# } -# to_bed(out_file, val0, properties=properties0) -# with open_bed_cloud(out_file) as bed1: -# assert np.allclose(val0, bed1.read(), equal_nan=True) -# val_sparse = bed1.read_sparse() -# assert np.allclose(val0, val_sparse.toarray(), equal_nan=True) -# assert np.array_equal(bed.fid, properties0["fid"]) -# assert np.array_equal(bed.iid, properties0["iid"]) -# assert np.array_equal(bed.sid, properties0["sid"]) -# assert np.issubdtype(bed.sid.dtype, np.str_) -# assert np.array_equal(bed.chromosome, properties0["chromosome"]) -# assert np.allclose(bed.cm_position, properties0["cm_position"]) -# assert np.allclose(bed.bp_position, properties0["bp_position"]) - -# val_float = val0.astype("float") -# val_float[0, 0] = 0.5 - -# for force_python_only in [False, True]: -# with pytest.raises(ValueError): -# to_bed( -# out_file, -# val_float, -# properties=properties0, -# force_python_only=force_python_only, -# ) -# val0[np.isnan(val0)] = 0 # set any nan to 0 -# val_int8 = val0.astype("int8") -# val_int8[0, 0] = -1 -# for force_python_only in [False, True]: -# with pytest.raises(ValueError): -# to_bed( -# out_file, -# val_int8, -# properties=properties0, -# force_python_only=force_python_only, -# ) +@pytest.mark.asyncio +async def test_cloud_write(tmp_path, shared_datadir): + in_file = shared_datadir / "plink_sim_10s_100v_10pmiss.bed" + out_file = tmp_path / "out.bed" + async with await open_bed_cloud.create(in_file) as bed_cloud: + val0 = await bed_cloud.read() + properties0 = { + "fid": await bed_cloud.fid(), + "iid": await bed_cloud.iid(), + "sid": await bed_cloud.sid(), + "chromosome": await bed_cloud.chromosome(), + "cm_position": await bed_cloud.cm_position(), + "bp_position": await bed_cloud.bp_position(), + } + # cmk write + to_bed(out_file, val0, properties=properties0) + async with await open_bed_cloud.create(out_file) as bed_cloud_1: + assert np.allclose(val0, await bed_cloud_1.read(), equal_nan=True) + assert np.array_equal(await bed_cloud_1.fid(), properties0["fid"]) + assert np.array_equal(await bed_cloud_1.iid(), properties0["iid"]) + assert np.array_equal(await bed_cloud_1.sid(), properties0["sid"]) + assert np.issubdtype((await bed_cloud_1.sid()).dtype, np.str_) + assert np.array_equal(await bed_cloud_1.chromosome(), properties0["chromosome"]) + assert np.allclose(await bed_cloud_1.cm_position(), properties0["cm_position"]) + assert np.allclose(await bed_cloud_1.bp_position(), properties0["bp_position"]) + + # val_float = val0.astype("float") + # val_float[0, 0] = 0.5 + + # for force_python_only in [False, True]: + # with pytest.raises(ValueError): + # to_bed( + # out_file, + # val_float, + # properties=properties0, + # force_python_only=force_python_only, + # ) + # val0[np.isnan(val0)] = 0 # set any nan to 0 + # val_int8 = val0.astype("int8") + # val_int8[0, 0] = -1 + # for force_python_only in [False, True]: + # with pytest.raises(ValueError): + # to_bed( + # out_file, + # val_int8, + # properties=properties0, + # force_python_only=force_python_only, + # ) def test_cloud_overrides(shared_datadir): diff --git a/src/python_module.rs b/src/python_module.rs index 9d4dcc7..8e2cbdb 100644 --- a/src/python_module.rs +++ b/src/python_module.rs @@ -63,7 +63,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { let ii = &iid_index.as_slice()?; let si = &sid_index.as_slice()?; - let mut val = unsafe { val.as_array_mut() }; + let mut val = val.readwrite(); + let mut val = val.as_array_mut(); let mut bed = Bed::builder(filename) .iid_count(iid_count) @@ -98,7 +99,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { let ii = &iid_index.as_slice()?; let si = &sid_index.as_slice()?; - let mut val = unsafe { val.as_array_mut() }; + let mut val = val.readwrite(); + let mut val = val.as_array_mut(); let mut bed = Bed::builder(filename) .iid_count(iid_count) @@ -133,7 +135,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { let ii = &iid_index.as_slice()?; let si = &sid_index.as_slice()?; - let mut val = unsafe { val.as_array_mut() }; + let mut val = val.readwrite(); + let mut val = val.as_array_mut(); let mut bed = Bed::builder(filename) .iid_count(iid_count) @@ -175,7 +178,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { // // let ii = &iid_index.as_slice()?; // // let si = &sid_index.as_slice()?; - // // let mut val = unsafe { val.as_array_mut() }; + // // let mut val = val.readwrite(); + // // let mut val = val.as_array_mut(); // // // cmk hack @@ -211,7 +215,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { val: &PyArray2, num_threads: usize, ) -> Result<(), PyErr> { - let val = unsafe { val.as_array() }; + let mut val = val.readwrite(); + let mut val = val.as_array_mut(); WriteOptions::builder(filename) .is_a1_counted(is_a1_counted) @@ -231,7 +236,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { val: &PyArray2, num_threads: usize, ) -> Result<(), PyErr> { - let val = unsafe { val.as_array() }; + let mut val = val.readwrite(); + let mut val = val.as_array_mut(); WriteOptions::builder(filename) .is_a1_counted(is_a1_counted) @@ -251,7 +257,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { val: &PyArray2, num_threads: usize, ) -> Result<(), PyErr> { - let val = unsafe { val.as_array() }; + let mut val = val.readwrite(); + let mut val = val.as_array_mut(); WriteOptions::builder(filename) .is_a1_counted(is_a1_counted) @@ -274,8 +281,11 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { ) -> Result<(), PyErr> { let iid_index = iid_index.readonly(); let sid_index = sid_index.readonly(); - let val_in = unsafe { val_in.as_array() }; - let mut val_out = unsafe { val_out.as_array_mut() }; + + let val_in = val_in.readwrite(); + let val_in = val_in.as_array(); + let mut val_out = val_out.readwrite(); + let mut val_out = val_out.as_array_mut(); let ii = &iid_index.as_slice()?; let si = &sid_index.as_slice()?; @@ -297,8 +307,11 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { ) -> Result<(), PyErr> { let iid_index = iid_index.readonly(); let sid_index = sid_index.readonly(); - let val_in = unsafe { val_in.as_array() }; - let mut val_out = unsafe { val_out.as_array_mut() }; + + let val_in = val_in.readwrite(); + let val_in = val_in.as_array(); + let mut val_out = val_out.readwrite(); + let mut val_out = val_out.as_array_mut(); let ii = &iid_index.as_slice()?; let si = &sid_index.as_slice()?; @@ -320,8 +333,11 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { ) -> Result<(), PyErr> { let iid_index = iid_index.readonly(); let sid_index = sid_index.readonly(); - let val_in = unsafe { val_in.as_array() }; - let mut val_out = unsafe { val_out.as_array_mut() }; + + let val_in = val_in.readwrite(); + let val_in = val_in.as_array(); + let mut val_out = val_out.readwrite(); + let mut val_out = val_out.as_array_mut(); let ii = &iid_index.as_slice()?; let si = &sid_index.as_slice()?; @@ -345,8 +361,10 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { stats: &PyArray2, num_threads: usize, ) -> Result<(), PyErr> { - let mut val = unsafe { val.as_array_mut() }; - let mut stats = unsafe { stats.as_array_mut() }; + let mut val = val.readwrite(); + let mut val = val.as_array_mut(); + let mut stats = stats.readwrite(); + let mut stats = stats.as_array_mut(); let dist = create_dist(beta_not_unit_variance, beta_a, beta_b); create_pool(num_threads)?.install(|| { impute_and_zero_mean_snps( @@ -381,8 +399,10 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { stats: &PyArray2, num_threads: usize, ) -> Result<(), PyErr> { - let mut val = unsafe { val.as_array_mut() }; - let mut stats = unsafe { stats.as_array_mut() }; + let mut val = val.readwrite(); + let mut val = val.as_array_mut(); + let mut stats = stats.readwrite(); + let mut stats = stats.as_array_mut(); let dist = create_dist(beta_not_unit_variance, beta_a, beta_b); create_pool(num_threads)?.install(|| { @@ -410,7 +430,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { num_threads: usize, log_frequency: usize, ) -> Result<(), PyErr> { - let mut ata_piece = unsafe { ata_piece.as_array_mut() }; + let mut ata_piece = ata_piece.readwrite(); + let mut ata_piece = ata_piece.as_array_mut(); create_pool(num_threads)?.install(|| { file_ata_piece( @@ -441,7 +462,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { num_threads: usize, log_frequency: usize, ) -> Result<(), PyErr> { - let mut ata_piece = unsafe { ata_piece.as_array_mut() }; + let mut ata_piece = ata_piece.readwrite(); + let mut ata_piece = ata_piece.as_array_mut(); create_pool(num_threads)?.install(|| { file_ata_piece( @@ -471,7 +493,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { num_threads: usize, log_frequency: usize, ) -> Result<(), PyErr> { - let mut ata_piece = unsafe { ata_piece.as_array_mut() }; + let mut ata_piece = ata_piece.readwrite(); + let mut ata_piece = ata_piece.as_array_mut(); create_pool(num_threads)?.install(|| { _file_ata_piece_internal( @@ -501,7 +524,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { num_threads: usize, log_frequency: usize, ) -> Result<(), PyErr> { - let mut aat_piece = unsafe { aat_piece.as_array_mut() }; + let mut aat_piece = aat_piece.readwrite(); + let mut aat_piece = aat_piece.as_array_mut(); create_pool(num_threads)?.install(|| { file_aat_piece( @@ -532,7 +556,8 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { num_threads: usize, log_frequency: usize, ) -> Result<(), PyErr> { - let mut aat_piece = unsafe { aat_piece.as_array_mut() }; + let mut aat_piece = aat_piece.readwrite(); + let mut aat_piece = aat_piece.as_array_mut(); create_pool(num_threads)?.install(|| { file_aat_piece( @@ -563,9 +588,12 @@ fn bed_reader(_py: Python<'_>, m: &PyModule) -> PyResult<()> { num_threads: usize, log_frequency: usize, ) -> Result<(), PyErr> { - let mut b1 = unsafe { b1.as_array_mut() }; - let mut aatb = unsafe { aatb.as_array_mut() }; - let mut atb = unsafe { atb.as_array_mut() }; + let mut b1 = b1.readwrite(); + let mut b1 = b1.as_array_mut(); + let mut aatb = aatb.readwrite(); + let mut aatb = aatb.as_array_mut(); + let mut atb = atb.readwrite(); + let mut atb = atb.as_array_mut(); create_pool(num_threads)?.install(|| { file_b_less_aatbx(