feat: update to netcdf 0.8.1 and add rustler precompiled (#6)

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
   workflow_dispatch:
 
 env:
-  EXPLORER_BUILD: true
+  NETCDF_BUILD: true
   RUST_TOOLCHAIN_VERSION: nightly-2022-08-16
   MIX_ENV: test
 
@@ -61,4 +61,4 @@ jobs:
         with:
           otp-version: 25.0
           elixir-version: 1.14
-      - run: mix format --check-formatted
+      - run: mix format --check-formatted

.github/workflows/release.yml

@@ -0,0 +1,69 @@
+name: Elixir CI
+on:
+  release:
+    types: [published]
+
+env:
+  RUST_TOOLCHAIN_VERSION: nightly-2022-08-16
+  MIX_ENV: prod
+  NETCDF_BUILD: true
+
+jobs:
+  build_release:
+    name: Release ${{ matrix.nif }} - ${{ matrix.job.target }}
+    runs-on: ${{ matrix.job.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        nif: ["2.15", "2.16"]
+        job:
+          - { target: x86_64-apple-darwin, os: macos-11 }
+          - { target: x86_64-unknown-linux-gnu, os: ubuntu-20.04 }
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Conda
+        uses: s-weigand/setup-conda@v1
+        with:
+          conda-channels: conda-forge
+      - name: Extract project version
+        shell: bash
+        run: |
+          # Get the project version from mix.exs
+          echo "PROJECT_VERSION=$(sed -n 's/^  @version "\(.*\)"/\1/p' mix.exs | head -n1)" >> $GITHUB_ENV
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+          target: ${{ matrix.job.target }}
+      - name: Install dependencies
+        if: ${{ matrix.job.os != 'macos-11' }}
+        run: |
+          conda install -y -c conda-forge libnetcdf=4.8.1 hdf5=1.12.1
+          echo "HDF5_DIR=${CONDA_PREFIX}"
+          echo "NETCDF_DIR=${CONDA_PREFIX}" >> $GITHUB_ENV
+          echo "RUSTFLAGS=-C link-args=-Wl,-rpath,$CONDA_PREFIX/lib" >> $GITHUB_ENV
+      - name: Install dependencies (Mac)
+        if: ${{ matrix.job.os == 'macos-11' }}
+        run: brew install nco
+      - name: Build the project
+        id: build-crate
+        uses: philss/[email protected]
+        with:
+          project-name: ex_netcdf
+          project-version: ${{ env.PROJECT_VERSION }}
+          target: ${{ matrix.job.target }}
+          nif-version: ${{ matrix.nif }}
+          use-cross: ${{ matrix.job.use-cross }}
+          project-dir: "native/ex_netcdf"
+      - name: Artifact upload
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ steps.build-crate.outputs.file-name }}
+          path: ${{ steps.build-crate.outputs.file-path }}
+      - name: Publish archives and packages
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            ${{ steps.build-crate.outputs.file-path }}
+        if: startsWith(github.ref, 'refs/tags/')

.gitignore

@@ -61,6 +61,9 @@ erl_crash.dump
 
 /priv
 /priv/native/libex_netcdf
+/release_build
+*.so.tar.gz
+
 
 ### Elixir Patch ###
 

Dockerfile.gnu-linux

@@ -0,0 +1,87 @@
+# Find eligible builder and runner images on Docker Hub. We use Ubuntu/Debian
+# instead of Alpine to avoid DNS resolution issues in production.
+#
+# https://hub.docker.com/r/hexpm/elixir/tags?page=1&name=ubuntu
+# https://hub.docker.com/_/ubuntu?tab=tags
+#
+# This file is based on these images:
+#
+#   - https://hub.docker.com/r/hexpm/elixir/tags - for the build image
+#   - https://hub.docker.com/_/debian?tab=tags&page=1&name=bullseye-20210902-slim - for the release image
+#   - https://pkgs.org/ - resource for finding needed packages
+#   - Ex: hexpm/elixir:1.14.2-erlang-25.0.2-debian-bullseye-20210902-slim
+#
+ARG ELIXIR_VERSION=1.14.2
+ARG OTP_VERSION=25.0.2
+ARG DEBIAN_VERSION=bullseye-20210902-slim
+
+ARG BUILDER_IMAGE="hexpm/elixir:${ELIXIR_VERSION}-erlang-${OTP_VERSION}-debian-${DEBIAN_VERSION}"
+ARG RUNNER_IMAGE="debian:${DEBIAN_VERSION}"
+
+ARG RUSTLER_NIF_VERSION
+
+FROM ${BUILDER_IMAGE} as builder
+
+# install build dependencies
+RUN apt-get update -y && apt-get install -y cmake build-essential git curl wget \
+  && apt-get clean && rm -f /var/lib/apt/lists/*_*
+
+# install Rust non-interactively, and put it in the $PATH so the netcdf Elixir dep can find it
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.66.1
+ENV PATH="/root/.cargo/bin:$PATH"
+
+ENV CONDA_PREFIX=/opt/conda
+ENV PATH=${CONDA_PREFIX}/bin:$PATH
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-$(arch).sh -O ~/miniconda.sh && /bin/bash ~/miniconda.sh -b -p ${CONDA_PREFIX}
+RUN conda install -y -c conda-forge libnetcdf=4.8.1 hdf5=1.12.1
+ENV HDF5_DIR=${CONDA_PREFIX}
+ENV NETCDF_DIR=${CONDA_PREFIX}
+ENV RUSTFLAGS="-C link-args=-Wl,-rpath,$CONDA_PREFIX/lib"
+ENV NETCDF_BUILD=true
+
+ENV MIX_ENV=prod
+
+# prepare build dir
+WORKDIR /app
+
+# install hex + rebar
+RUN mix local.hex --force && \
+  mix local.rebar --force
+
+# install mix dependencies
+COPY mix.exs mix.lock VERSION ./
+RUN mix deps.get --only prod
+RUN mkdir config
+
+# copy compile-time config files before we compile dependencies
+# to ensure any relevant config change will trigger the dependencies
+# to be re-compiled.
+# COPY config/config.exs config/prod.exs config/
+RUN mix deps.compile
+
+COPY lib lib
+COPY native native
+
+# Compile the release
+RUN mix compile
+
+# Changes to config/runtime.exs don't require recompiling the code
+# COPY config/runtime.exs config/
+
+ENV RUSTLER_NIF_VERSION=${RUSTLER_NIF_VERSION}
+RUN mix release
+
+FROM ${RUNNER_IMAGE}
+
+WORKDIR "/app"
+
+COPY --from=builder --chown=nobody:nogroup /app/_build/prod/rel/netcdf ./
+
+RUN mkdir /mnt/release
+RUN chown nobody:nogroup /mnt/release
+RUN chown nobody:nogroup /app
+
+USER nobody
+
+
+CMD ["/bin/bash"]

Makefile

@@ -0,0 +1,70 @@
+# Usage:
+# Run `make` or `make all` to generate all combinations of OS=linux/mac and NIF_VERSION=2.15/2.16
+# for the host architecture. Mostly useful for aarch64 because CI takes care of x86_64 releases.
+
+VALID_NIF_VERSIONS := 2.15 2.16
+
+VERSION_PATTERN := ^[0-9]+\.[0-9]+\.[0-9]+$$
+
+# Set the VERSION environment variable using the contents of the VERSION file
+VERSION := $(shell cat VERSION)
+
+# Check that the VERSION variable matches the expected pattern
+ifeq ($(shell echo $(VERSION) | grep -Eo '$(VERSION_PATTERN)'),$(VERSION))
+  $(info VERSION is valid: '$(VERSION)')
+else
+  $(error VERSION is invalid: '$(VERSION)')
+endif
+
+ifeq ($(shell uname -s), Darwin)
+TARGET_OS := apple-darwin
+else
+TARGET_OS := unknown-linux-gnu
+endif
+
+ifeq ($(shell uname -m), arm64)
+# Processor is Mac arm
+TARGET_ARCH := aarch64
+else
+# Processor is not Mac arm
+TARGET_ARCH := x86_64
+endif
+
+NETCDF_SO_PREFIX := libex_netcdf-v$(VERSION)-nif-$(NIF_VERSION)-$(TARGET_ARCH)
+NETCDF_SO := $(NETCDF_SO_PREFIX)-$(TARGET_OS).so
+GNU_NETCDF_SO := $(NETCDF_SO_PREFIX)-unknown-linux-gnu.so
+
+.PHONY: all
+all:
+	@echo "Building targets for NIF versions: $(VALID_NIF_VERSIONS) and arch: $(TARGET_ARCH)"
+	@$(foreach nif_version,$(VALID_NIF_VERSIONS), \
+		$(MAKE) gnu-docker-release NIF_VERSION=$(nif_version); \
+		$(MAKE) host-release NIF_VERSION=$(nif_version); \
+	)
+
+.PHONY: validate_nif_env
+validate_nif_env:
+	@if [ -z "$$NIF_VERSION" ]; then \
+        echo "NIF_VERSION is not set"; \
+        exit 1; \
+	fi
+	@if [ -z "$(filter ${NIF_VERSION}, ${VALID_NIF_VERSIONS})" ]; then \
+		echo "NIF_VERSION is set to ${NIF_VERSION}, but it should be one of: ${VALID_NIF_VERSIONS}"; \
+		exit 1; \
+	fi
+	@echo "NIF_VERSION is set to ${NIF_VERSION} and is valid"
+
+.PHONY: gnu-docker-release
+gnu-docker-release: validate_nif_env
+	docker build --tag netcdf-release --build-arg RUSTLER_NIF_VERSION=$(NIF_VERSION) -f Dockerfile.gnu-linux .
+	docker run --mount type=bind,source="$(shell pwd)",target=/mnt/release netcdf-release cp /app/lib/netcdf-$(VERSION)/priv/native/libex_netcdf.so /mnt/release/$(GNU_NETCDF_SO)
+	tar -czvf $(GNU_NETCDF_SO).tar.gz $(GNU_NETCDF_SO)
+	rm $(GNU_NETCDF_SO)
+
+.PHONY: host-release
+host-release: validate_nif_env
+	mix deps.get --only=prod
+	RUSTLER_NIF_VERSION=$(NIF_VERSION) MIX_ENV=prod NETCDF_BUILD=true mix release --overwrite --force
+	mv _build/prod/rel/netcdf/lib/netcdf-$(VERSION)/priv/native/libex_netcdf.so $(NETCDF_SO)
+	tar -czvf $(NETCDF_SO).tar.gz $(NETCDF_SO)
+	rm $(NETCDF_SO)

README.md

@@ -11,26 +11,46 @@ by adding `netcdf` to your list of dependencies in `mix.exs`:
 ```elixir
 def deps do
   [
-    {:netcdf, "~> 0.1.0"}
+    {:netcdf, "~> 0.2.0"}
   ]
 end
 ```
 
 ### Dependencies
 
+For usage, you don't need any dependencies installed.
+The library uses `:rustler_precompiled` to fetch compiled versions for the following targets:
+
+- x86_64-apple-darwin
+- x86_64-unknown-linux-gnu
+- aarch64-apple-darwin
+- aarch64-unknown-linux-gnu
+
+If you want to compile locally or if you are running on an unsupported target,
+you must set `NETCDF_BUILD=true` in your environment, and follow the instructions below.
+
 Before compiling, you also must ensure that the following dependencies are installed on your system:
 
-- hdf5
-- libnetcdf
+- hdf5@1.12.1
+- libnetcdf@4.8.1
 
 On Ubuntu:
 
-`apt install ibhdf5-serial-dev libnetcdf-dev`
+`apt install libhdf5-serial-dev libnetcdf-dev`
 
 On macOS:
 
 `brew install netcdf-cxx`, which will also bring `hdf5` as a dependency
 
+Conda can also be used to install the dependencies instead:
+
+```shell
+conda install -y -c conda-forge libnetcdf=4.8.1 hdf5=1.12.1
+echo "HDF5_DIR=${CONDA_PREFIX}"
+echo "NETCDF_DIR=${CONDA_PREFIX}" >> $GITHUB_ENV
+echo "RUSTFLAGS=-C link-args=-Wl,-rpath,$CONDA_PREFIX/lib" >> $GITHUB_ENV
+```
+
 ### Utilities
 
 Although not necessary for the library to work, `ncks` is helpful for downloading `.nc` files in the correct way,

VERSION

@@ -0,0 +1 @@
+0.2.0

lib/netcdf/native.ex

@@ -1,6 +1,26 @@
 defmodule NetCDF.Native do
   @moduledoc false
-  use Rustler, otp_app: :netcdf, crate: "ex_netcdf"
+
+  mode = if Mix.env() in [:dev, :test], do: :debug, else: :release
+  force_build = mode == :debug or System.get_env("NETCDF_BUILD") in ["1", "true"]
+
+  mix_config = Mix.Project.config()
+  version = mix_config[:version]
+  github_url = mix_config[:package][:links]["GitHub"]
+
+  use RustlerPrecompiled,
+    otp_app: :netcdf,
+    crate: "ex_netcdf",
+    base_url: "#{github_url}/releases/download/v#{version}",
+    version: version,
+    force_build: force_build,
+    targets: [
+      "aarch64-unknown-linux-gnu",
+      "aarch64-apple-darwin",
+      "x86_64-apple-darwin",
+      "x86_64-unknown-linux-gnu"
+    ],
+    mode: mode
 
   # netcdf::file
   def file_open(_filename), do: :erlang.nif_error(:nif_not_loaded)

mix.exs

@@ -1,8 +1,8 @@
-defmodule DataParser.MixProject do
+defmodule NetCDF.MixProject do
   use Mix.Project
 
   @source_url "https://github.com/DockYard/netcdf"
-  @version "0.1.1"
+  @version File.read!("VERSION")
 
   def project do
     [
@@ -32,7 +32,8 @@ defmodule DataParser.MixProject do
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
-      {:rustler, "~> 0.26.0"},
+      {:rustler_precompiled, "~> 0.6.1"},
+      {:rustler, "~> 0.26.0", optional: true},
       {:ex_doc, "~> 0.29.0", only: :docs}
     ]
   end

mix.lock

@@ -1,4 +1,5 @@
 %{
+  "castore": {:hex, :castore, "1.0.1", "240b9edb4e9e94f8f56ab39d8d2d0a57f49e46c56aced8f873892df8ff64ff5a", [:mix], [], "hexpm", "b4951de93c224d44fac71614beabd88b71932d0b1dea80d2f80fb9044e01bbb3"},
   "complex": {:hex, :complex, "0.4.2", "923e5db0be13dbb3ea00cf8459d9f75f3afdd9ff5a82742ded21064330d28273", [:mix], [], "hexpm", "069a085ef820ce675a2619fd125b963ff4514af2102c7f7d7965128e5ec0a429"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.29", "149d50dcb3a93d9f3d6f3ecf18c918fb5a2d3c001b5d3305c926cddfbd33355b", [:mix], [], "hexpm", "4902af1b3eb139016aed210888748db8070b8125c2342ce3dcae4f38dcc63503"},
   "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"},
@@ -11,5 +12,6 @@
   "nimble_parsec": {:hex, :nimble_parsec, "1.2.3", "244836e6e3f1200c7f30cb56733fd808744eca61fd182f731eac4af635cc6d0b", [:mix], [], "hexpm", "c8d789e39b9131acf7b99291e93dae60ab48ef14a7ee9d58c6964f59efb570b0"},
   "nx": {:hex, :nx, "0.3.0", "12448769619ed1442e22bd198faf4629ae3d0ee3ed565e42be49829cc42bf35f", [:mix], [{:complex, "~> 0.4.2", [hex: :complex, repo: "hexpm", optional: false]}], "hexpm", "d08d8962f4379ade54281aad84c45cb7eb0118b406fc836f07b94acc40df5859"},
   "rustler": {:hex, :rustler, "0.26.0", "06a2773d453ee3e9109efda643cf2ae633dedea709e2455ac42b83637c9249bf", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:toml, "~> 0.6", [hex: :toml, repo: "hexpm", optional: false]}], "hexpm", "42961e9d2083d004d5a53e111ad1f0c347efd9a05cb2eb2ffa1d037cdc74db91"},
+  "rustler_precompiled": {:hex, :rustler_precompiled, "0.6.1", "160b545bce8bf9a3f1b436b2c10f53574036a0db628e40f393328cbbe593602f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "0dd269fa261c4e3df290b12031c575fff07a542749f7b0e8b744d72d66c43600"},
   "toml": {:hex, :toml, "0.6.2", "38f445df384a17e5d382befe30e3489112a48d3ba4c459e543f748c2f25dd4d1", [:mix], [], "hexpm", "d013e45126d74c0c26a38d31f5e8e9b83ea19fc752470feb9a86071ca5a672fa"},
 }