diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 407b978..d5df362 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: workflow_dispatch: env: - EXPLORER_BUILD: true + NETCDF_BUILD: true RUST_TOOLCHAIN_VERSION: nightly-2022-08-16 MIX_ENV: test @@ -61,4 +61,4 @@ jobs: with: otp-version: 25.0 elixir-version: 1.14 - - run: mix format --check-formatted \ No newline at end of file + - run: mix format --check-formatted diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..8eeb70b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,69 @@ +name: Elixir CI +on: + release: + types: [published] + +env: + RUST_TOOLCHAIN_VERSION: nightly-2022-08-16 + MIX_ENV: prod + NETCDF_BUILD: true + +jobs: + build_release: + name: Release ${{ matrix.nif }} - ${{ matrix.job.target }} + runs-on: ${{ matrix.job.os }} + strategy: + fail-fast: false + matrix: + nif: ["2.15", "2.16"] + job: + - { target: x86_64-apple-darwin, os: macos-11 } + - { target: x86_64-unknown-linux-gnu, os: ubuntu-20.04 } + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Install Conda + uses: s-weigand/setup-conda@v1 + with: + conda-channels: conda-forge + - name: Extract project version + shell: bash + run: | + # Get the project version from mix.exs + echo "PROJECT_VERSION=$(sed -n 's/^ @version "\(.*\)"/\1/p' mix.exs | head -n1)" >> $GITHUB_ENV + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: stable + target: ${{ matrix.job.target }} + - name: Install dependencies + if: ${{ matrix.job.os != 'macos-11' }} + run: | + conda install -y -c conda-forge libnetcdf=4.8.1 hdf5=1.12.1 + echo "HDF5_DIR=${CONDA_PREFIX}" + echo "NETCDF_DIR=${CONDA_PREFIX}" >> $GITHUB_ENV + echo "RUSTFLAGS=-C link-args=-Wl,-rpath,$CONDA_PREFIX/lib" >> $GITHUB_ENV + - name: Install dependencies (Mac) + if: ${{ matrix.job.os == 'macos-11' }} + run: brew install nco + - name: Build the project + id: build-crate + uses: philss/rustler-precompiled-action@v1.0.0 + with: + project-name: ex_netcdf + project-version: ${{ env.PROJECT_VERSION }} + target: ${{ matrix.job.target }} + nif-version: ${{ matrix.nif }} + use-cross: ${{ matrix.job.use-cross }} + project-dir: "native/ex_netcdf" + - name: Artifact upload + uses: actions/upload-artifact@v3 + with: + name: ${{ steps.build-crate.outputs.file-name }} + path: ${{ steps.build-crate.outputs.file-path }} + - name: Publish archives and packages + uses: softprops/action-gh-release@v1 + with: + files: | + ${{ steps.build-crate.outputs.file-path }} + if: startsWith(github.ref, 'refs/tags/') diff --git a/.gitignore b/.gitignore index 2dce70b..d492a2f 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,9 @@ erl_crash.dump /priv /priv/native/libex_netcdf +/release_build +*.so.tar.gz + ### Elixir Patch ### diff --git a/Dockerfile.gnu-linux b/Dockerfile.gnu-linux new file mode 100644 index 0000000..8f8c33d --- /dev/null +++ b/Dockerfile.gnu-linux @@ -0,0 +1,87 @@ +# Find eligible builder and runner images on Docker Hub. We use Ubuntu/Debian +# instead of Alpine to avoid DNS resolution issues in production. +# +# https://hub.docker.com/r/hexpm/elixir/tags?page=1&name=ubuntu +# https://hub.docker.com/_/ubuntu?tab=tags +# +# This file is based on these images: +# +# - https://hub.docker.com/r/hexpm/elixir/tags - for the build image +# - https://hub.docker.com/_/debian?tab=tags&page=1&name=bullseye-20210902-slim - for the release image +# - https://pkgs.org/ - resource for finding needed packages +# - Ex: hexpm/elixir:1.14.2-erlang-25.0.2-debian-bullseye-20210902-slim +# +ARG ELIXIR_VERSION=1.14.2 +ARG OTP_VERSION=25.0.2 +ARG DEBIAN_VERSION=bullseye-20210902-slim + +ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}" +ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}" + +ARG RUSTLER_NIF_VERSION + +FROM ${BUILDER_IMAGE} as builder + +# install build dependencies +RUN apt-get update -y && apt-get install -y cmake build-essential git curl wget \ + && apt-get clean && rm -f /var/lib/apt/lists/*_* + +# install Rust non-interactively, and put it in the $PATH so the netcdf Elixir dep can find it +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.66.1 +ENV PATH="/root/.cargo/bin:$PATH" + +ENV CONDA_PREFIX=/opt/conda +ENV PATH=${CONDA_PREFIX}/bin:$PATH +RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-$(arch).sh -O ~/miniconda.sh && /bin/bash ~/miniconda.sh -b -p ${CONDA_PREFIX} +RUN conda install -y -c conda-forge libnetcdf=4.8.1 hdf5=1.12.1 +ENV HDF5_DIR=${CONDA_PREFIX} +ENV NETCDF_DIR=${CONDA_PREFIX} +ENV RUSTFLAGS="-C link-args=-Wl,-rpath,$CONDA_PREFIX/lib" +ENV NETCDF_BUILD=true + +ENV MIX_ENV=prod + +# prepare build dir +WORKDIR /app + +# install hex + rebar +RUN mix local.hex --force && \ + mix local.rebar --force + +# install mix dependencies +COPY mix.exs mix.lock VERSION ./ +RUN mix deps.get --only prod +RUN mkdir config + +# copy compile-time config files before we compile dependencies +# to ensure any relevant config change will trigger the dependencies +# to be re-compiled. +# COPY config/config.exs config/prod.exs config/ +RUN mix deps.compile + +COPY lib lib +COPY native native + +# Compile the release +RUN mix compile + +# Changes to config/runtime.exs don't require recompiling the code +# COPY config/runtime.exs config/ + +ENV RUSTLER_NIF_VERSION=${RUSTLER_NIF_VERSION} +RUN mix release + +FROM ${RUNNER_IMAGE} + +WORKDIR "/app" + +COPY --from=builder --chown=nobody:nogroup /app/_build/prod/rel/netcdf ./ + +RUN mkdir /mnt/release +RUN chown nobody:nogroup /mnt/release +RUN chown nobody:nogroup /app + +USER nobody + + +CMD ["/bin/bash"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..22fb3fa --- /dev/null +++ b/Makefile @@ -0,0 +1,70 @@ +# Usage: +# Run `make` or `make all` to generate all combinations of OS=linux/mac and NIF_VERSION=2.15/2.16 +# for the host architecture. Mostly useful for aarch64 because CI takes care of x86_64 releases. + +VALID_NIF_VERSIONS := 2.15 2.16 + +VERSION_PATTERN := ^[0-9]+\.[0-9]+\.[0-9]+$$ + +# Set the VERSION environment variable using the contents of the VERSION file +VERSION := $(shell cat VERSION) + +# Check that the VERSION variable matches the expected pattern +ifeq ($(shell echo $(VERSION) | grep -Eo '$(VERSION_PATTERN)'),$(VERSION)) + $(info VERSION is valid: '$(VERSION)') +else + $(error VERSION is invalid: '$(VERSION)') +endif + +ifeq ($(shell uname -s), Darwin) +TARGET_OS := apple-darwin +else +TARGET_OS := unknown-linux-gnu +endif + +ifeq ($(shell uname -m), arm64) +# Processor is Mac arm +TARGET_ARCH := aarch64 +else +# Processor is not Mac arm +TARGET_ARCH := x86_64 +endif + +NETCDF_SO_PREFIX := libex_netcdf-v$(VERSION)-nif-$(NIF_VERSION)-$(TARGET_ARCH) +NETCDF_SO := $(NETCDF_SO_PREFIX)-$(TARGET_OS).so +GNU_NETCDF_SO := $(NETCDF_SO_PREFIX)-unknown-linux-gnu.so + +.PHONY: all +all: + @echo "Building targets for NIF versions: $(VALID_NIF_VERSIONS) and arch: $(TARGET_ARCH)" + @$(foreach nif_version,$(VALID_NIF_VERSIONS), \ + $(MAKE) gnu-docker-release NIF_VERSION=$(nif_version); \ + $(MAKE) host-release NIF_VERSION=$(nif_version); \ + ) + +.PHONY: validate_nif_env +validate_nif_env: + @if [ -z "$$NIF_VERSION" ]; then \ + echo "NIF_VERSION is not set"; \ + exit 1; \ + fi + @if [ -z "$(filter ${NIF_VERSION}, ${VALID_NIF_VERSIONS})" ]; then \ + echo "NIF_VERSION is set to ${NIF_VERSION}, but it should be one of: ${VALID_NIF_VERSIONS}"; \ + exit 1; \ + fi + @echo "NIF_VERSION is set to ${NIF_VERSION} and is valid" + +.PHONY: gnu-docker-release +gnu-docker-release: validate_nif_env + docker build --tag netcdf-release --build-arg RUSTLER_NIF_VERSION=$(NIF_VERSION) -f Dockerfile.gnu-linux . + docker run --mount type=bind,source="$(shell pwd)",target=/mnt/release netcdf-release cp /app/lib/netcdf-$(VERSION)/priv/native/libex_netcdf.so /mnt/release/$(GNU_NETCDF_SO) + tar -czvf $(GNU_NETCDF_SO).tar.gz $(GNU_NETCDF_SO) + rm $(GNU_NETCDF_SO) + +.PHONY: host-release +host-release: validate_nif_env + mix deps.get --only=prod + RUSTLER_NIF_VERSION=$(NIF_VERSION) MIX_ENV=prod NETCDF_BUILD=true mix release --overwrite --force + mv _build/prod/rel/netcdf/lib/netcdf-$(VERSION)/priv/native/libex_netcdf.so $(NETCDF_SO) + tar -czvf $(NETCDF_SO).tar.gz $(NETCDF_SO) + rm $(NETCDF_SO) \ No newline at end of file diff --git a/README.md b/README.md index 2215dad..82376fb 100644 --- a/README.md +++ b/README.md @@ -11,26 +11,46 @@ by adding `netcdf` to your list of dependencies in `mix.exs`: ```elixir def deps do [ - {:netcdf, "~> 0.1.0"} + {:netcdf, "~> 0.2.0"} ] end ``` ### Dependencies +For usage, you don't need any dependencies installed. +The library uses `:rustler_precompiled` to fetch compiled versions for the following targets: + +- x86_64-apple-darwin +- x86_64-unknown-linux-gnu +- aarch64-apple-darwin +- aarch64-unknown-linux-gnu + +If you want to compile locally or if you are running on an unsupported target, +you must set `NETCDF_BUILD=true` in your environment, and follow the instructions below. + Before compiling, you also must ensure that the following dependencies are installed on your system: -- hdf5 -- libnetcdf +- hdf5@1.12.1 +- libnetcdf@4.8.1 On Ubuntu: -`apt install ibhdf5-serial-dev libnetcdf-dev` +`apt install libhdf5-serial-dev libnetcdf-dev` On macOS: `brew install netcdf-cxx`, which will also bring `hdf5` as a dependency +Conda can also be used to install the dependencies instead: + +```shell +conda install -y -c conda-forge libnetcdf=4.8.1 hdf5=1.12.1 +echo "HDF5_DIR=${CONDA_PREFIX}" +echo "NETCDF_DIR=${CONDA_PREFIX}" >> $GITHUB_ENV +echo "RUSTFLAGS=-C link-args=-Wl,-rpath,$CONDA_PREFIX/lib" >> $GITHUB_ENV +``` + ### Utilities Although not necessary for the library to work, `ncks` is helpful for downloading `.nc` files in the correct way, diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..341cf11 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.2.0 \ No newline at end of file diff --git a/lib/netcdf/native.ex b/lib/netcdf/native.ex index 08684b5..2c43c5b 100644 --- a/lib/netcdf/native.ex +++ b/lib/netcdf/native.ex @@ -1,6 +1,26 @@ defmodule NetCDF.Native do @moduledoc false - use Rustler, otp_app: :netcdf, crate: "ex_netcdf" + + mode = if Mix.env() in [:dev, :test], do: :debug, else: :release + force_build = mode == :debug or System.get_env("NETCDF_BUILD") in ["1", "true"] + + mix_config = Mix.Project.config() + version = mix_config[:version] + github_url = mix_config[:package][:links]["GitHub"] + + use RustlerPrecompiled, + otp_app: :netcdf, + crate: "ex_netcdf", + base_url: "#{github_url}/releases/download/v#{version}", + version: version, + force_build: force_build, + targets: [ + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "x86_64-unknown-linux-gnu" + ], + mode: mode # netcdf::file def file_open(_filename), do: :erlang.nif_error(:nif_not_loaded) diff --git a/mix.exs b/mix.exs index 40bdfa6..eb053c4 100644 --- a/mix.exs +++ b/mix.exs @@ -1,8 +1,8 @@ -defmodule DataParser.MixProject do +defmodule NetCDF.MixProject do use Mix.Project @source_url "https://github.com/DockYard/netcdf" - @version "0.1.1" + @version File.read!("VERSION") def project do [ @@ -32,7 +32,8 @@ defmodule DataParser.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:rustler, "~> 0.26.0"}, + {:rustler_precompiled, "~> 0.6.1"}, + {:rustler, "~> 0.26.0", optional: true}, {:ex_doc, "~> 0.29.0", only: :docs} ] end diff --git a/mix.lock b/mix.lock index 06401ce..be6e4be 100644 --- a/mix.lock +++ b/mix.lock @@ -1,4 +1,5 @@ %{ + "castore": {:hex, :castore, "1.0.1", "240b9edb4e9e94f8f56ab39d8d2d0a57f49e46c56aced8f873892df8ff64ff5a", [:mix], [], "hexpm", "b4951de93c224d44fac71614beabd88b71932d0b1dea80d2f80fb9044e01bbb3"}, "complex": {:hex, :complex, "0.4.2", "923e5db0be13dbb3ea00cf8459d9f75f3afdd9ff5a82742ded21064330d28273", [:mix], [], "hexpm", "069a085ef820ce675a2619fd125b963ff4514af2102c7f7d7965128e5ec0a429"}, "earmark_parser": {:hex, :earmark_parser, "1.4.29", "149d50dcb3a93d9f3d6f3ecf18c918fb5a2d3c001b5d3305c926cddfbd33355b", [:mix], [], "hexpm", "4902af1b3eb139016aed210888748db8070b8125c2342ce3dcae4f38dcc63503"}, "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"}, @@ -11,5 +12,6 @@ "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"}, "nx": {:hex, :nx, "0.3.0", "12448769619ed1442e22bd198faf4629ae3d0ee3ed565e42be49829cc42bf35f", [:mix], [{:complex, "~> 0.4.2", [hex: :complex, repo: "hexpm", optional: false]}], "hexpm", "d08d8962f4379ade54281aad84c45cb7eb0118b406fc836f07b94acc40df5859"}, "rustler": {:hex, :rustler, "0.26.0", "06a2773d453ee3e9109efda643cf2ae633dedea709e2455ac42b83637c9249bf", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "42961e9d2083d004d5a53e111ad1f0c347efd9a05cb2eb2ffa1d037cdc74db91"}, + "rustler_precompiled": {:hex, :rustler_precompiled, "0.6.1", "160b545bce8bf9a3f1b436b2c10f53574036a0db628e40f393328cbbe593602f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "0dd269fa261c4e3df290b12031c575fff07a542749f7b0e8b744d72d66c43600"}, "toml": {:hex, :toml, "0.6.2", "38f445df384a17e5d382befe30e3489112a48d3ba4c459e543f748c2f25dd4d1", [:mix], [], "hexpm", "d013e45126d74c0c26a38d31f5e8e9b83ea19fc752470feb9a86071ca5a672fa"}, } diff --git a/native/ex_netcdf/Cargo.lock b/native/ex_netcdf/Cargo.lock index 723b6bb..615bfda 100644 --- a/native/ex_netcdf/Cargo.lock +++ b/native/ex_netcdf/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "aho-corasick" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f55bd91a0978cbfd91c457a164bab8b4001c833b7f323132c0a4e1922dd44e" +checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" dependencies = [ "memchr", ] @@ -25,9 +25,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "cc" -version = "1.0.73" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" [[package]] name = "cfg-if" @@ -35,6 +35,30 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + +[[package]] +name = "curl-sys" +version = "0.4.61+curl-8.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14d05c10f541ae6f3bc5b3d923c20001f47db7d5f0b2bc6ad16490133842db79" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", + "winapi", +] + [[package]] name = "ex_netcdf" version = "0.1.0" @@ -45,14 +69,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "hdf5-src" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01493db39ddc0519cf2a83d620d2c037fee60f4fed724cb72dc23763f1727a8" +dependencies = [ + "cmake", + "libz-sys", +] + [[package]] name = "hdf5-sys" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4842d5980dc311a7c8933c7b45534fdae84df5ae7939a0ae8e449a56d4beb3d2" dependencies = [ + "hdf5-src", "libc", "libloading", + "libz-sys", "pkg-config", "regex", "serde", @@ -62,9 +98,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "lazy_static" @@ -74,15 +110,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.135" +version = "0.2.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68783febc7782c6c5cb401fbda4de5a9898be1762314da0bb2c10ced61f18b0c" +checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" [[package]] name = "libloading" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbc0f03f9a775e9f6aed295c6a1ba2253c5757a9e03d55c6caa46a681abcddd" +checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" dependencies = [ "cfg-if", "winapi", @@ -130,9 +166,9 @@ dependencies = [ [[package]] name = "netcdf" -version = "0.7.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9668c493289156ed0dce9472ea33ab7a6f6dfe4565a8433bd103995f8053b467" +checksum = "dfea96e04fb6c7cc3e58d735dbf9213e30b7e88aac0349feeab66abbcb826acc" dependencies = [ "bitflags", "lazy_static", @@ -140,21 +176,35 @@ dependencies = [ "netcdf-sys", ] +[[package]] +name = "netcdf-src" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efdec1c75d5b0adad74b499c3d97a3906cedd1a9a0dae02a9b8547baef79ec3" +dependencies = [ + "cmake", + "hdf5-sys", + "libz-sys", +] + [[package]] name = "netcdf-sys" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1028362b1f88b5c4bdf3538b202b2c83ab3cc8a5537f6337ac147e6e36dc5f9" +checksum = "33907cddc6881b45f9106ff87aa02eaeb362855826e02765357320a5151a34da" dependencies = [ + "curl-sys", "hdf5-sys", "libz-sys", + "netcdf-src", + "semver", ] [[package]] name = "num-complex" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19" +checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" dependencies = [ "num-traits", ] @@ -178,26 +228,49 @@ dependencies = [ "autocfg", ] +[[package]] +name = "openssl-src" +version = "111.25.2+1.1.1t" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320708a054ad9b3bf314688b5db87cf4d6683d64cfc835e2337924ae62bf4431" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666416d899cf077260dac8698d60a60b435a46d57e82acb1be3d0dad87284e5b" +dependencies = [ + "autocfg", + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + [[package]] name = "pkg-config" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "e472a104799c74b514a57226160104aa483546de37e839ec50e3c2e41dd87534" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.21" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ "proc-macro2", ] @@ -210,9 +283,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" [[package]] name = "regex" -version = "1.6.0" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b" +checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d" dependencies = [ "aho-corasick", "memchr", @@ -221,9 +294,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.27" +version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "rustler" @@ -245,7 +318,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 1.0.109", ] [[package]] @@ -258,28 +331,45 @@ dependencies = [ "unreachable", ] +[[package]] +name = "semver" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" + [[package]] name = "serde" -version = "1.0.145" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728eb6351430bccb993660dfffc5a72f91ccc1295abaa8ce19b27ebe4f75568b" +checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065" [[package]] name = "serde_derive" -version = "1.0.145" +version = "1.0.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fa1584d3d1bcacd84c277a0dfe21f5b0f6accf4a23d04d4c6d61f1af522b4c" +checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.12", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] name = "syn" -version = "1.0.102" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fcd952facd492f9be3ef0d0b7032a6e442ee9b361d4acc2b1d0c4aaa5f613a1" +checksum = "79d9531f94112cfc3e4c8f5f02cb2b58f72c97b7efd85f70203cc6d8efda5927" dependencies = [ "proc-macro2", "quote", @@ -288,29 +378,29 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.12", ] [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unreachable" diff --git a/native/ex_netcdf/Cargo.toml b/native/ex_netcdf/Cargo.toml index 7877338..266c240 100644 --- a/native/ex_netcdf/Cargo.toml +++ b/native/ex_netcdf/Cargo.toml @@ -11,6 +11,6 @@ crate-type = ["cdylib"] [dependencies] rustler = "0.26.0" -netcdf = "0.7.0" +netcdf = { version = "0.8.1", features = ["static"] } thiserror = "1" -ndarray = "0.15.6" +ndarray = "0.15.6" \ No newline at end of file diff --git a/native/ex_netcdf/src/lib.rs b/native/ex_netcdf/src/lib.rs index 4870605..c4bfbbc 100644 --- a/native/ex_netcdf/src/lib.rs +++ b/native/ex_netcdf/src/lib.rs @@ -1,4 +1,5 @@ use netcdf::attribute::Attribute; +use netcdf::extent::{Extents, Extents::All}; use netcdf::types::{BasicType, VariableType}; use rustler::{Env, Term}; @@ -24,6 +25,7 @@ rustler::atoms! { u64_t = "u64", f32_t = "f32", f64_t = "f64", + string_t = "string" } fn on_load(env: Env, _info: Term) -> bool { @@ -68,52 +70,57 @@ fn variable_values( fn get_variable_values( variable: &netcdf::variable::Variable, ) -> Result<(Value, rustler::types::atom::Atom), NetCDFError> { + match variable.vartype() { + VariableType::Basic(BasicType::Byte) => load_numeric_variable_values::(variable), + VariableType::Basic(BasicType::Char) => load_numeric_variable_values::(variable), + VariableType::Basic(BasicType::Ubyte) => load_numeric_variable_values::(variable), + VariableType::Basic(BasicType::Short) => load_numeric_variable_values::(variable), + VariableType::Basic(BasicType::Ushort) => load_numeric_variable_values::(variable), + VariableType::Basic(BasicType::Int) => load_numeric_variable_values::(variable), + VariableType::Basic(BasicType::Uint) => load_numeric_variable_values::(variable), + VariableType::Basic(BasicType::Int64) => load_numeric_variable_values::(variable), + VariableType::Basic(BasicType::Uint64) => load_numeric_variable_values::(variable), + VariableType::Basic(BasicType::Float) => load_numeric_variable_values::(variable), + VariableType::Basic(BasicType::Double) => load_numeric_variable_values::(variable), + VariableType::String => load_string_variable_values(variable), + _ => Err(error::NetCDFError::NetCDF( + netcdf::error::Error::WrongDataset, + )), + } +} + +fn load_numeric_variable_values( + variable: &netcdf::variable::Variable, +) -> Result<(Value, rustler::types::atom::Atom), NetCDFError> +where + Value: From>, + T: netcdf::NcPutGet, +{ let var_type = variable.vartype(); let type_name = var_type.name(); + let error = netcdf::error::Error::Str(format!("unable to load type {}", type_name)); - let values = match var_type { - VariableType::Basic(BasicType::Byte) => variable - .values::(None, None) - .map(|x| Value::from(x.into_raw_vec())), - VariableType::Basic(BasicType::Char) => variable - .values::(None, None) - .map(|x| Value::from(x.into_raw_vec())), - VariableType::Basic(BasicType::Ubyte) => variable - .values::(None, None) - .map(|x| Value::from(x.into_raw_vec())), - VariableType::Basic(BasicType::Short) => variable - .values::(None, None) - .map(|x| Value::from(x.into_raw_vec())), - VariableType::Basic(BasicType::Ushort) => variable - .values::(None, None) - .map(|x| Value::from(x.into_raw_vec())), - VariableType::Basic(BasicType::Int) => variable - .values::(None, None) - .map(|x| Value::from(x.into_raw_vec())), - VariableType::Basic(BasicType::Uint) => variable - .values::(None, None) - .map(|x| Value::from(x.into_raw_vec())), - VariableType::Basic(BasicType::Int64) => variable - .values::(None, None) - .map(|x| Value::from(x.into_raw_vec())), - VariableType::Basic(BasicType::Uint64) => variable - .values::(None, None) - .map(|x| Value::from(x.into_raw_vec())), - VariableType::Basic(BasicType::Float) => variable - .values::(None, None) - .map(|x| Value::from(x.into_raw_vec())), - VariableType::Basic(BasicType::Double) => variable - .values::(None, None) - .map(|x| Value::from(x.into_raw_vec())), - _ => Err(netcdf::error::Error::Str(format!( - "unable to load type {}", - type_name - ))), - }; + let value = variable + .values::(All) + .map_err(|_| error::NetCDFError::NetCDF(error))?; + Ok((Value::from(value), as_type_atom(&type_name))) +} + +fn load_string_variable_values( + variable: &netcdf::variable::Variable, +) -> Result<(Value, rustler::types::atom::Atom), NetCDFError> { + let time_len = variable.len() as usize; + let mut values: Vec = Vec::with_capacity(time_len); + + for i in 0..time_len { + let error = netcdf::error::Error::Str("unable to load string variable".to_string()); + let value: String = variable + .string_value::(i.into()) + .map_err(|_| error::NetCDFError::NetCDF(error))?; + values.push(value); + } - values - .map(|result| (result, as_type_atom(&type_name))) - .map_err(NetCDFError::NetCDF) + Ok((Value::from(values), string_t())) } fn as_type_atom(type_name: &str) -> rustler::types::atom::Atom { diff --git a/priv/string_test.nc b/priv/string_test.nc new file mode 100644 index 0000000..fdf97c1 Binary files /dev/null and b/priv/string_test.nc differ diff --git a/test/netcdf/variable_test.exs b/test/netcdf/variable_test.exs index 5e507fc..c56bc46 100644 --- a/test/netcdf/variable_test.exs +++ b/test/netcdf/variable_test.exs @@ -47,5 +47,22 @@ defmodule NetCDF.VariableTest do ) end end + + test "loads string variable" do + filename = Path.join(:code.priv_dir(:netcdf), "string_test.nc") + {:ok, file} = File.open(filename) + + assert %File{filename: ^filename, resource: _, variables: ["string_var"]} = file + + assert {:ok, + %Variable{ + name: "string_var", + value: ~w(Hello world ! This is a test), + type: :string, + attributes: attributes + }} = Variable.load(file, "string_var") + + assert %{} == attributes + end end end