feat: support reproducible compilation and install (#6)

This PR update the `pyproject.toml`, `Manifest.in` and `Cargo.toml` to support compilation and installation of rust `outlines_core`. Note* building and install relies on `pip` rather then the `build` library. --------- Co-authored-by: erikkaum <[email protected]>
dottxt-ai · Aug 19, 2024 · 2066b1e · 2066b1e
1 parent 99de33c
commit 2066b1e
Show file tree

Hide file tree

Showing 9 changed files with 76 additions and 40 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,4 +8,7 @@ docs/build
 .venv
 benchmarks/results
 **/target/
-**/dist/
+**/dist/
+
+bindings/python/build
+bindings/python/src/outlines_core
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/README.md b/README.md
@@ -7,9 +7,25 @@
 
 ## developing
 
-- build only the outlines-core package `cd outlines-core && cargo build`
-- dev build of python bindings `cd bindings/python && maturin develop`. If you have the conda `outlines-dev` environment activated, the outlines-core module is installed within the env automatically
+There's a [justfile](https://github.com/casey/just) for most dev & build tasks
 
-There's also a [justfile](https://github.com/casey/just) for running these easier:
-- `just dev-core`
-- `just dev-python`
+- build only the outlines-core rust crate `cd outlines-core && cargo build`
+- install an editable pip package with the recepie `just dev-python` which is:
+```bash
+cd bindings/python && pip install -e .
+```
+- to build the python package, run `just build-python`, which is equivalent to:
+```bash
+cd bindings/python && \
+ln -sf ../../outlines-core outlines-core-lib && \
+sed -i '' 's|path = "../../outlines-core"|path = "outlines-core-lib"|' Cargo.toml && \
+python -m build && \
+rm outlines-core-lib && \
+sed -i '' 's|path = "outlines-core-lib"|path = "../../outlines-core"|' Cargo.toml
+```
+
+### Developer Notes
+
+- Setup a virtual environment before running the build or dev commands
+
+- If you get the `LookupError: setuptools-scm was unable to detect version for...` error, set the env var `SETUPTOOLS_SCM_PRETEND_VERSION=0.1.0-dev` before running the build or dev command.
diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml
@@ -5,8 +5,6 @@ edition = "2021"
 
 [dependencies]
 pyo3 = "0.22.0"
-# outlines-core = { path = "../../outlines-core" }
-
 [profile.release-lto]
 inherits = "release"
 lto = true
@@ -15,3 +13,6 @@ lto = true
 name = "_lib"
 crate-type = ["cdylib"]
 path = "rust/lib.rs"
+
+[dependencies.outlines-core]
+path = "../../outlines-core"
diff --git a/bindings/python/Manifest.in b/bindings/python/Manifest.in
@@ -1,6 +1,9 @@
-graft python
+graft src 
 graft rust
-graft tests
 include Cargo.toml
+
 global-exclude */__pycache__/*
 global-exclude *.pyc
+
+recursive-include outlines-core-lib *
+recursive-exclude outlines-core/target *
diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml
@@ -4,11 +4,14 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "outlines_core"
-authors= [{name = "Outlines Developers"}]
+authors = [
+    { name = "Outlines Developers" },
+    { name = "Huggingface Developers" },
+]
 description = "Structured Text Generation in Rust"
 requires-python = ">=3.8"
-license = {text = "Apache-2.0"}
-keywords=[
+license = { text = "Apache-2.0" }
+keywords = [
     "machine learning",
     "deep learning",
     "language models",
@@ -24,17 +27,17 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
 dependencies = [
-   "interegular",
-   "numpy<2.0.0",
-   "cloudpickle",
-   "diskcache",
-   "pydantic>=2.0",
-   "numba",
-   "referencing",
-   "jsonschema",
-   "tqdm",
-   "datasets",
-   "typing_extensions",
+    "interegular",
+    "numpy<2.0.0",
+    "cloudpickle",
+    "diskcache",
+    "pydantic>=2.0",
+    "numba",
+    "referencing",
+    "jsonschema",
+    "tqdm",
+    "datasets",
+    "typing_extensions",
 ]
 dynamic = ["version"]
 
@@ -61,7 +64,7 @@ documentation = "https://outlines-dev.github.io/outlines-core/"
 repository = "https://github.com/outlines-dev/outlines-core/"
 
 [project.readme]
-file="README.md"
+file = "README.md"
 content-type = "text/markdown"
 
 [tool.setuptools]
@@ -109,26 +112,17 @@ module = [
 ignore_missing_imports = true
 
 [tool.coverage.run]
-omit = [
-    "src/outlines_core/_version.py",
-    "tests/*",
-]
+omit = ["src/outlines_core/_version.py", "tests/*"]
 branch = true
 
 [tool.coverage.report]
-omit = [
-    "tests/*",
-]
-exclude_lines = [
-    "pragma: no cover",
-    "if TYPE_CHECKING:",
-    "\\.\\.\\.",
-]
+omit = ["tests/*"]
+exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "\\.\\.\\."]
 show_missing = true
 
 [tool.diff_cover]
 compare_branch = "origin/main"
 diff_range_notation = ".."
 
 [[tool.setuptools-rust.ext-modules]]
-target = "outlines_core._lib"  # The last part of the name (e.g. "_lib") has to match lib.name in Cargo.toml, but you can add a prefix to nest it inside of a Python package.
+target = "outlines_core._lib" # The last part of the name (e.g. "_lib") has to match lib.name in Cargo.toml, but you can add a prefix to nest it inside of a Python package.
diff --git a/bindings/python/rust/lib.rs b/bindings/python/rust/lib.rs
@@ -4,16 +4,26 @@
 // use pyo3::types::{PyAnyMethods, PyDict, PyModule, PyModuleMethods, PySet};
 use pyo3::{pyfunction, pymodule, wrap_pyfunction, Bound, PyResult};
 
-
 #[pymodule]
 mod _lib {
+    use outlines_core::FLAG;
     use pyo3::prelude::*;
 
     /// Formats the sum of two numbers as string.
     #[pyfunction]
     fn sum_as_string(a: usize, b: usize) -> PyResult<String> {
         Ok((a + b).to_string())
     }
+
+    #[pyfunction]
+    fn show_me_the_flag() -> PyResult<String> {
+        Ok(FLAG.to_string())
+    }
+
+    #[pyfunction]
+    fn anotherone() -> PyResult<String> {
+        Ok("This is another one".to_string())
+    }
 }
 
 // #[pymodule]

diff --git a/justfile b/justfile
@@ -8,4 +8,10 @@ dev-python:
     cd bindings/python && pip install -e .
 
 build-python:
-    cd bindings/python && python -m build
+    cd bindings/python && \
+    ln -sf ../../outlines-core outlines-core-lib && \
+    sed -i '' 's|path = "../../outlines-core"|path = "outlines-core-lib"|' Cargo.toml && \
+    python -m build && \
+    rm outlines-core-lib && \
+    sed -i '' 's|path = "outlines-core-lib"|path = "../../outlines-core"|' Cargo.toml
+
diff --git a/outlines-core/src/lib.rs b/outlines-core/src/lib.rs
@@ -3,6 +3,8 @@ use std::collections::BTreeSet;
 use std::sync::Arc;
 use std::thread;
 
+pub const FLAG: bool = true;
+
 pub fn create_fsm_index_end_to_end_rust(
     fsm_transitions: &BTreeMap<(i32, i32), i32>,
     alphabet_symbol_mapping: &BTreeMap<char, i32>,