diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1b7ef1..bc3f39d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,8 +55,10 @@ jobs: distro: ubuntu22.04 githubToken: ${{ github.token }} install: | - apt-get update - apt-get install -y --no-install-recommends python3 python3-pip + apt update + apt install -y --no-install-recommends \ + gcc g++ gfortran libopenblas-dev liblapack-dev ninja-build \ + pkg-config python3-pip python3-dev pip3 install -U pip pytest run: | set -e diff --git a/Cargo.lock b/Cargo.lock index 9dcf8b2..ae43326 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "autocfg" version = "1.1.0" @@ -14,6 +20,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" + [[package]] name = "cfg-if" version = "1.0.0" @@ -24,7 +36,46 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" name = "cog3pio" version = "0.1.0" dependencies = [ + "ndarray", + "numpy", "pyo3", + "tempfile", + "tiff", +] + +[[package]] +name = "crc32fast" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +dependencies = [ + "crc32fast", + "miniz_oxide", ] [[package]] @@ -39,12 +90,24 @@ version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "libc" version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + [[package]] name = "lock_api" version = "0.4.11" @@ -55,6 +118,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "matrixmultiply" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7574c1cf36da4798ab73da5b215bbf444f50718207754cb522201d78d1cd0ff2" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "memoffset" version = "0.9.0" @@ -64,6 +137,70 @@ dependencies = [ "autocfg", ] +[[package]] +name = "miniz_oxide" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +dependencies = [ + "adler", +] + +[[package]] +name = "ndarray" +version = "0.15.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "rawpointer", +] + +[[package]] +name = "num-complex" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23c6602fda94a57c990fe0df199a035d83576b496aa29f4e634a8ac6004e68a6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "numpy" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef41cbb417ea83b30525259e30ccef6af39b31c240bda578889494c5392d331" +dependencies = [ + "libc", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pyo3", + "rustc-hash", +] + [[package]] name = "once_cell" version = "1.19.0" @@ -90,7 +227,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets", + "windows-targets 0.48.5", ] [[package]] @@ -180,13 +317,38 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "redox_syscall" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "bitflags", + "bitflags 1.3.2", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +dependencies = [ + "bitflags 2.4.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", ] [[package]] @@ -218,6 +380,29 @@ version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys", +] + +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "unicode-ident" version = "1.0.12" @@ -230,19 +415,49 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.3", +] + [[package]] name = "windows-targets" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f" +dependencies = [ + "windows_aarch64_gnullvm 0.52.3", + "windows_aarch64_msvc 0.52.3", + "windows_i686_gnu 0.52.3", + "windows_i686_msvc 0.52.3", + "windows_x86_64_gnu 0.52.3", + "windows_x86_64_gnullvm 0.52.3", + "windows_x86_64_msvc 0.52.3", ] [[package]] @@ -251,38 +466,80 @@ version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +[[package]] +name = "windows_i686_gnu" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb" + [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +[[package]] +name = "windows_i686_msvc" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6" diff --git a/Cargo.toml b/Cargo.toml index 77e3252..5d16a2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,4 +10,10 @@ name = "cog3pio" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.20.3", features = ["abi3-py310"] } +ndarray = "0.15.6" +numpy = "0.20.0" +pyo3 = { version = "0.20.3", features = ["abi3-py310", "extension-module"] } +tiff = "0.9.1" + +[dev-dependencies] +tempfile = "3.10.1" diff --git a/pyproject.toml b/pyproject.toml index 5b5fe0a..3b631f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,9 @@ classifiers = [ "License :: OSI Approved :: Apache Software License", "License :: OSI Approved :: MIT License", ] +dependencies = [ + "numpy>=1.23", +] dynamic = ["version"] [project.optional-dependencies] diff --git a/python/cog3pio/__init__.py b/python/cog3pio/__init__.py index 303368a..b1217f8 100644 --- a/python/cog3pio/__init__.py +++ b/python/cog3pio/__init__.py @@ -4,7 +4,7 @@ from importlib.metadata import version -from .cog3pio import * +from .cog3pio import read_geotiff __doc__ = cog3pio.__doc__ __version__ = version("cog3pio") # e.g. 0.1.2.dev3+g0ab3cd78 diff --git a/python/tests/test_io_geotiff.py b/python/tests/test_io_geotiff.py new file mode 100644 index 0000000..eec96bb --- /dev/null +++ b/python/tests/test_io_geotiff.py @@ -0,0 +1,32 @@ +""" +Test I/O on GeoTIFF files. +""" +import tempfile +import urllib.request +import os +import pytest + +from cog3pio import read_geotiff + + +@pytest.fixture(scope="module", name="geotiff_path") +def fixture_geotiff_path(): + """ + Filepath to a sample single-band GeoTIFF file. + """ + with tempfile.TemporaryDirectory() as tmpdir: + geotiff_path = os.path.join(tmpdir, "float32.tif") + urllib.request.urlretrieve( + url="https://github.com/pka/georaster/raw/v0.1.0/data/tiff/float32.tif", + filename=geotiff_path, + ) + yield geotiff_path + + +def test_read_geotiff(geotiff_path): + """ + Read a GeoTIFF file from a local file path. + """ + array = read_geotiff(path=geotiff_path) + assert array.shape == (20, 20) + assert array.dtype == "float32" diff --git a/src/io/geotiff.rs b/src/io/geotiff.rs new file mode 100644 index 0000000..b7ddd95 --- /dev/null +++ b/src/io/geotiff.rs @@ -0,0 +1,62 @@ +use std::io::{Read, Seek}; + +use ndarray::{Array2, ShapeError}; +use tiff::decoder::{DecodingResult, Limits}; + +/// Synchronously read a GeoTIFF file into an [`ndarray::Array`] +pub fn read_geotiff(stream: R) -> Result, ShapeError> { + // Open TIFF stream with decoder + let mut decoder = tiff::decoder::Decoder::new(stream).expect("Cannot create tiff decoder"); + decoder = decoder.with_limits(Limits::unlimited()); + + // Get image dimensions + let dimensions: (u32, u32) = decoder.dimensions().expect("Cannot parse image dimensions"); + let width = dimensions.0 as usize; + let height = dimensions.1 as usize; + + // Get image pixel data + let DecodingResult::F32(img_data) = decoder.read_image().expect("Cannot decode tiff image") + else { + panic!("Cannot read band data") + }; + + // Put image pixel data into an ndarray + let vec_data = Array2::from_shape_vec((height, width), img_data)?; + + Ok(vec_data) +} + +#[cfg(test)] +mod tests { + use std::io::{Seek, SeekFrom}; + + use tempfile::tempfile; + use tiff::encoder::{colortype, TiffEncoder}; + + use crate::io::geotiff::read_geotiff; + + #[test] + fn test_read_geotiff() { + // Generate some data + let mut image_data = Vec::new(); + for x in 0..20 { + for y in 0..20 { + let val = x + y; + image_data.push(val as f32); + } + } + + // Write a BigTIFF file + let mut file = tempfile().unwrap(); + let mut bigtiff = TiffEncoder::new_big(&mut file).unwrap(); + bigtiff + .write_image::(20, 20, &image_data) + .unwrap(); + file.seek(SeekFrom::Start(0)).unwrap(); + + // Read a BigTIFF file + let arr = read_geotiff(file).unwrap(); + assert_eq!(arr.dim(), (20, 20)); + assert_eq!(arr.mean(), Some(19.0)); + } +} diff --git a/src/io/mod.rs b/src/io/mod.rs new file mode 100644 index 0000000..179f176 --- /dev/null +++ b/src/io/mod.rs @@ -0,0 +1,2 @@ +/// Read and write GeoTIFF files +pub mod geotiff; diff --git a/src/lib.rs b/src/lib.rs index 5d9c3f9..7530a31 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,54 @@ -use pyo3::prelude::{pymodule, PyModule, PyResult, Python}; +#![warn(missing_docs)] +//! # Cloud-optimized GeoTIFF ... Parallel I/O +//! +//! A reader for [Cloud Optimized GeoTIFF (COG)](https://www.cogeo.org) files. +//! +//! Uses [`tiff`] to decode TIFF images, storing the pixel data in [`ndarray`] structs. +//! +//! **Note**: For Python users, there are also bindings (via [`pyo3`]) to read GeoTIFF files into +//! `numpy.ndarray` objects (i.e. similar to [`rasterio`](https://github.com/rasterio/rasterio)). +//! This is done via the [`numpy`] crate which enables passing data from Rust to Python. -/// A Python module implemented in Rust. +/// Modules for handling Input/Output of GeoTIFF data +pub mod io; + +use std::fs::File; + +use ndarray::Dim; +use numpy::{PyArray, ToPyArray}; +use pyo3::prelude::{pyfunction, pymodule, PyModule, PyResult, Python}; +use pyo3::wrap_pyfunction; + +/// Read a GeoTIFF file from a path on disk into an ndarray +/// +/// Parameters +/// ---------- +/// path : str +/// The path to the file. +/// +/// Returns +/// ------- +/// array : np.ndarray +/// 2D array containing the GeoTIFF pixel data. +#[pyfunction] +#[pyo3(name = "read_geotiff")] +fn read_geotiff_py<'py>( + path: &str, + py: Python<'py>, +) -> PyResult<&'py PyArray>> { + // Open TIFF file from path + let file = File::open(path).expect("Cannot find GeoTIFF file"); + // Get image pixel data as an ndarray + let vec_data = io::geotiff::read_geotiff(file).expect("Cannot read GeoTIFF"); + // Convert from ndarray (Rust) to numpy ndarray (Python) + Ok(vec_data.to_pyarray(py)) +} + +/// A Python module implemented in Rust. The name of this function must match +/// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to +/// import the module. #[pymodule] fn cog3pio(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(read_geotiff_py, m)?)?; Ok(()) }